.

django/finances/electrum.py

@@ -4,11 +4,12 @@ from connectrum import ElectrumErrorResponse
 import json
 import asyncio
 import threading
-import logging
+
 from labelbase.models import Label
 #from labelbase.utils import compute_type_ref_hash
 from finances.models import OutputStat, HistoricalPrice
 
+import logging
 logger = logging.getLogger('labelbase')
 
 
@@ -60,9 +61,6 @@ async def interact_addr(conn, server_info, method, addr):
 def checkup_label(label_id, loop):
     if label_id and loop:
         """
-        TODO:
-
-        [ ] electrum > network endpoint from settings
         [ ] get_or_create() OutputStat using unspent payload
         """
         elem = Label.objects.get(id=label_id)
@@ -71,8 +69,14 @@ def checkup_label(label_id, loop):
             output = OutputStat(type_ref_hash=elem.type_ref_hash, network=elem.labelbase.network, value=0)
         logger.debug("Output found {}".format(output.id))
         if elem.type == "output" and output.spent is not True:
-            logger.debug("Processing Label {}".format(label_id))
-            server_info = ServerInfo("bitcoin.lu.ke", "bitcoin.lu.ke", ports=(("s50002")))
+            electrum_hostname = elem.labelbase.user.profile.electrum_hostname
+            if not electrum_hostname:
+                electrum_hostname = "bitcoin.lu.ke"
+            electrum_ports = elem.labelbase.user.profile.electrum_ports
+            if not electrum_ports:
+                electrum_ports = "s50002"
+            logger.debug("Processing Label {} using electrum server connection: {} {} {}".format(label_id, electrum_hostname, electrum_hostname, electrum_ports))
+            server_info = ServerInfo(electrum_hostname, electrum_hostname, ports=((electrum_ports)))
             conn = StratumClient()
             logger.debug("type_ref_hash: {}".format(elem.type_ref_hash))
             assert elem.type_ref_hash
@@ -99,8 +103,6 @@ def checkup_label(label_id, loop):
                         utxo_height = unspent.get('height')
                         logger.debug("found {}".format(unspent))
                         break
-
-
                 if output:
                     output.network = elem.labelbase.network
                     if utxo_height:
@@ -119,7 +121,6 @@ def checkup_label(label_id, loop):
                     logger.debug("saved output {}".format(output.id))
         else:
             logger.debug("Label is not an output. Spent is {}".format(output.spent))
-
     else:
         if not label_id:
             logger.error("Can't get label_id! {}".format(label_id))

django/finances/models.py

@@ -8,6 +8,9 @@ from labelbase.receivers import compute_type_ref_hash
 from django.conf import settings
 
 
+import logging
+logger = logging.getLogger('labelbase')
+
 
 class OutputStat(models.Model):
     """
@@ -70,6 +73,9 @@ class OutputStat(models.Model):
             obj, created = HistoricalPrice.get_or_create_from_api(
                 timestamp=self.confirmed_at_block_time
             )
+            if obj is None:
+                logger.error("No price info found for {}".format(self.confirmed_at_block_time))
+                return {}
             print ("obj, created  = {} {}, self.confirmed_at_block_time {}".format(obj, created, self.confirmed_at_block_time))
 
             # Use tracked fiat value if available, otherwise estimate using past price
@@ -262,19 +268,22 @@ class HistoricalPrice(models.Model):
         url = f"https://mempool.space/api/v1/historical-price?timestamp={timestamp}"
         response = requests.get(url)
         api_response = response.json()
-        obj, created = cls.objects.get_or_create(timestamp=timestamp, defaults={
-            'usd_price': Decimal(str(api_response['prices'][0]['USD'])),
-            'eur_price': Decimal(str(api_response['prices'][0]['EUR'])),
-            'gbp_price': Decimal(str(api_response['prices'][0]['GBP'])),
-            'cad_price': Decimal(str(api_response['prices'][0]['CAD'])),
-            'chf_price': Decimal(str(api_response['prices'][0]['CHF'])),
-            'aud_price': Decimal(str(api_response['prices'][0]['AUD'])),
-            'jpy_price': Decimal(str(api_response['prices'][0]['JPY'])),
-            'usd_to_eur': Decimal(str(api_response['exchangeRates']['USDEUR'])),
-            'usd_to_gbp': Decimal(str(api_response['exchangeRates']['USDGBP'])),
-            'usd_to_cad': Decimal(str(api_response['exchangeRates']['USDCAD'])),
-            'usd_to_chf': Decimal(str(api_response['exchangeRates']['USDCHF'])),
-            'usd_to_aud': Decimal(str(api_response['exchangeRates']['USDAUD'])),
-            'usd_to_jpy': Decimal(str(api_response['exchangeRates']['USDJPY']))
-        })
-        return obj, created
+        try:
+            obj, created = cls.objects.get_or_create(timestamp=timestamp, defaults={
+                'usd_price': Decimal(str(api_response['prices'][0]['USD'])),
+                'eur_price': Decimal(str(api_response['prices'][0]['EUR'])),
+                'gbp_price': Decimal(str(api_response['prices'][0]['GBP'])),
+                'cad_price': Decimal(str(api_response['prices'][0]['CAD'])),
+                'chf_price': Decimal(str(api_response['prices'][0]['CHF'])),
+                'aud_price': Decimal(str(api_response['prices'][0]['AUD'])),
+                'jpy_price': Decimal(str(api_response['prices'][0]['JPY'])),
+                'usd_to_eur': Decimal(str(api_response['exchangeRates']['USDEUR'])),
+                'usd_to_gbp': Decimal(str(api_response['exchangeRates']['USDGBP'])),
+                'usd_to_cad': Decimal(str(api_response['exchangeRates']['USDCAD'])),
+                'usd_to_chf': Decimal(str(api_response['exchangeRates']['USDCHF'])),
+                'usd_to_aud': Decimal(str(api_response['exchangeRates']['USDAUD'])),
+                'usd_to_jpy': Decimal(str(api_response['exchangeRates']['USDJPY']))
+            })
+            return obj, created
+        except:
+            return None, None

django/importer/forms.py

@@ -1,17 +1,9 @@
 from django import forms
-
-IMPORTER_CHOICES = (
-    ("BIP-0329", "BIP-329 .jsonl"),
-    #("BIP-0329-7z-enc" , "BIP-329 (encrypted) .7z"),
-    ("csv-bluewallet", "BlueWallet .csv"),
-    ("csv-bitbox", "BitBox .csv"),
-    ("pocket-accointing", "Pocket Accointing .csv")
-)
-
+from .models import IMPORTER_CHOICES
 
 class UploadFileForm(forms.Form):
     labelbase_id = forms.IntegerField(widget=forms.HiddenInput())
     import_type = forms.ChoiceField(
-        choices=IMPORTER_CHOICES #, widget=forms.HiddenInput()
+        choices=IMPORTER_CHOICES  
     )
     file = forms.FileField()

django/importer/views.py

@@ -1,183 +1,41 @@
 from django.http import HttpResponseRedirect
 from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
-import os
-import json
-from labelbase.models import Labelbase
-from labelbase.serializers import LabelSerializer
 from django.shortcuts import get_object_or_404
 from django.contrib import messages
-import decimal
-
+from labelbase.models import Labelbase
 
 from .forms import UploadFileForm
-from tempfile import NamedTemporaryFile
-
-EOLSTOP = [b"", "", None, "\n"]
-
-
-def handle_uploaded_file(f):
-    fp = NamedTemporaryFile(delete=False)
-    for chunk in f.chunks():
-        fp.write(chunk)
-    return fp
-
+from .tasks import process_uploaded_data
+from .models import UploadedData
 
 @login_required
 def upload_labels(request):
     """
     Used to import labels manually using files.
     """
-
     if request.method == "POST":
         form = UploadFileForm(request.POST, request.FILES)
         if form.is_valid():
-            imported_lables = 0
             labelbase = get_object_or_404(
                 Labelbase,
                 id=form.cleaned_data.get("labelbase_id", ""),
                 user_id=request.user.id,
             )
-            fp = handle_uploaded_file(request.FILES["file"])
-            fp.seek(0)
-            # BIP-0329
-            if form.cleaned_data.get("import_type", "") == "BIP-0329":
-                while True:
-                    buf = fp.readline()
-                    if buf in EOLSTOP:
-                        break
-                    data = json.loads(buf)
-                    data["labelbase"] = labelbase.id
-                    serializer = LabelSerializer(data=data)
-                    if serializer.is_valid():
-                        serializer.save()
-                        imported_lables += 1
-            elif form.cleaned_data.get("import_type", "") == "BIP-0329-7z-enc":
-                # TODO: Implement
-                pass
-            # Bitbox App
-            elif form.cleaned_data.get("import_type", "") == "csv-bitbox":
-                while True:
-                    buf = fp.readline()
-                    if buf in EOLSTOP:
-                        break
-                    try:
-                        buf = str(buf.decode("utf-8"))
-                        sbuf = buf.split(",")
-                        # Time,Type,Amount,Unit,Fee,Address,Transaction ID,Note
-                        for elem in [("tx", 6), ("addr", 5)]:
-                            data = {
-                                "type": elem[0],
-                                "ref": sbuf[elem[1]],
-                                "label": " ".join(sbuf[7:]),
-                            }
-                            data["labelbase"] = labelbase.id
-                            serializer = LabelSerializer(data=data)
-                            if serializer.is_valid():
-                                serializer.save()
-                                imported_lables += 1
-                            else:
-                                messages.add_message(
-                                    request,
-                                    messages.ERROR,
-                                    'Could not process line "{}".'.format(buf),
-                                )
-                    except Exception as ex:
-                        messages.add_message(
-                            request,
-                            messages.ERROR,
-                            'Could not process line "{}", {}.'.format(buf, ex),
-                        )
-            # Pocket Accointing
-            elif form.cleaned_data.get("import_type", "") == "pocket-accointing":
-                fp.close()
-                csv_file_path = fp.name
-                mempool_api = labelbase.get_mempool_api()
-                from .pocket import validate_csv_format, parse_csv_to_json
-                if validate_csv_format(csv_file_path):
-                    for item in parse_csv_to_json(csv_file_path):
-                        #label = "Got {} {} for {} {} in tx=\"{}\" ref=\"{} #Pocket\" ".format(item[0].get('outSellAmount'), item[1].get('inBuyAsset') , item[2].get('inBuyAmount'), item[2].get('inBuyAsset') , item[0].get('operationId'), item[1].get('operationId'))
-
-                        label = "Got {} {} for {:.2f} {} with reference: {} #Pocket".format(item[0].get('outSellAmount'), item[1].get('inBuyAsset'), decimal.Decimal(item[2].get('inBuyAmount')), item[2].get('inBuyAsset'), item[1].get('operationId'))
-                        txid = item[0].get('operationId')
-                        tx = mempool_api.get_transaction(txid)
-                        potential_utxos = []
-                        vouts = tx.get("vout", [])
-                        for i in range(len(vouts)):
-                            if vouts[i].get('value', 0) == decimal.Decimal(item[0].get('outSellAmount'))*100000000:
-                                potential_utxos.append("{}:{}".format(txid, i ))
-                        data = {}
-                        if len(potential_utxos) == 1:
-                            # label UTXO/output of tx
-                            data = {
-                                "type": "output",
-                                "ref": potential_utxos[0],
-                                "label": label,
-                            }
-                        if len(potential_utxos) > 1:
-                            # mark tx, add warning tag
-                            data = {
-                                "type": "tx",
-                                "ref": txid,
-                                "label": "{} {}".format(label, "#W001_UTXO_NOT_FOUND"),
-                            }
-                        if data:
-                            data["labelbase"] = labelbase.id
-                            serializer = LabelSerializer(data=data)
-                            if serializer.is_valid():
-                                serializer.save()
-                                imported_lables += 1
-                            else:
-                                messages.add_message(
-                                    request,
-                                    messages.ERROR,
-                                    'Could not process record "{}".'.format(item),
-                                )
-                else:
-                    print("ERROR") # TODO
-            # BlueWallet
-            elif form.cleaned_data.get("import_type", "") == "csv-bluewallet":
-                while True:
-                    buf = fp.readline()
-                    if buf in EOLSTOP:
-                        break
-                    try:
-                        buf = str(buf.decode("utf-8"))
-                        sbuf = buf.split(",")
-                        data = {
-                            "type": "tx",
-                            "ref": sbuf[1],
-                            "label": " ".join(sbuf[3:]),
-                        }
-                        data["labelbase"] = labelbase.id
-                        serializer = LabelSerializer(data=data)
-                        if serializer.is_valid():
-                            serializer.save()
-                            imported_lables += 1
-                        else:
-                            messages.add_message(
-                                request,
-                                messages.ERROR,
-                                'Could not process line "{}".'.format(buf),
-                            )
-                    except Exception as ex:
-                        messages.add_message(
-                            request,
-                            messages.ERROR,
-                            'Could not process line "{}", {}.'.format(buf, ex),
-                        )
-            else:
-                fp.close()
-                os.unlink(fp.name)
-                return HttpResponseRedirect("/failed/url/")
-            fp.close()
-            os.unlink(fp.name)
-            if imported_lables:
-                messages.add_message(
-                    request,
-                    messages.INFO,
-                    "Processed and imported {} labels.".format(imported_lables),
-                )
+            uploaded_data = UploadedData.objects.create(
+                user=request.user,
+                labelbase=labelbase,
+                import_type=form.cleaned_data.get("import_type", ""),
+                file=request.FILES["file"],
+            )
+            # Schedule the background task to process the uploaded data
+            process_uploaded_data(uploaded_data.id)
+            messages.add_message(
+                request,
+                messages.INFO,
+                "Task scheduled successfully.",
+#               "Task scheduled successfully. You will be notified upon completion.",
+            )
             return HttpResponseRedirect(labelbase.get_absolute_url())
     else:
         form = UploadFileForm()

django/labellabor/settings.py

@@ -1,6 +1,8 @@
 import os
-import sentry_sdk
-from sentry_sdk.integrations.django import DjangoIntegration
+
+#import sentry_sdk
+#from sentry_sdk.integrations.django import DjangoIntegration
+
 from pathlib import Path
 from configparser import RawConfigParser
 
@@ -36,20 +38,22 @@ if DEBUG:
 else:
     ALLOWED_HOSTS = [proj_config.get("internal", "allowed_host")]
 
-sentry_sdk.init(
-    dsn="https://3b833ae08ccc4ff68793e961fff4921c@o4504646963232768.ingest.sentry.io/4504646967361536",
-    integrations=[
-        DjangoIntegration(),
-    ],
-    # Set traces_sample_rate to 1.0 to capture 100%
-    # of transactions for performance monitoring.
-    # We recommend adjusting this value in production.
-    traces_sample_rate=1.0,
-    # If you wish to associate users to errors (assuming you are using
-    # django.contrib.auth) you may enable sending PII data.
-    send_default_pii=False,
-)
-sentry_sdk.set_tag("version", "0.23.1")
+SENTRY_DSN="https://3b833ae08ccc4ff68793e961fff4921c@o4504646963232768.ingest.sentry.io/4504646967361536"
+if False:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        integrations=[
+            DjangoIntegration(),
+        ],
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for performance monitoring.
+        # We recommend adjusting this value in production.
+        traces_sample_rate=1.0,
+        # If you wish to associate users to errors (assuming you are using
+        # django.contrib.auth) you may enable sending PII data.
+        send_default_pii=False,
+    )
+    sentry_sdk.set_tag("version", "0.23.1")
 
 LOGGING = {
     'version': 1,
@@ -118,6 +122,7 @@ MIDDLEWARE = [
     #"django.middleware.cache.FetchFromCacheMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    #"labellabor.middleware.SentryUserMiddleware",
 ]
 
 

django/labellabor/urls.py

@@ -8,7 +8,8 @@ from django.contrib.auth.decorators import login_required
 
 from userprofile.views import (ProfileView,
                                 ProfileAvatarUpdateView,
-                                ProfileCurrencyUpdateView)
+                                ProfileCurrencyUpdateView,
+                                ElectrumInfoUpdateView)
 from userprofile.views import APIKeyView
 
 
@@ -78,6 +79,11 @@ urlpatterns = [
         login_required(ProfileAvatarUpdateView.as_view()),
         name="userprofile_avatar",
     ),
+    path(
+        "account/userprofile-electrum/",
+        login_required(ElectrumInfoUpdateView.as_view()),
+        name="userprofile_electrum",
+    ),
     path(
         "account/userprofile-currency/",
         login_required(ProfileCurrencyUpdateView.as_view()),

django/labellabor/views.py

@@ -67,6 +67,7 @@ class LabelDeleteView(DeleteView):
     error_url = "/#failed"
 
     def post(self, request, *args, **kwargs):
+        dummy + yummy
         self.object = self.get_object()
         if self.object.labelbase.user != self.request.user:
             return redirect(self.error_url)
@@ -404,6 +405,8 @@ class FixAndMergeLabelsView(View):
         # Fetch all records for the specified Labelbase
         records = Label.objects.filter(labelbase_id=labelbase_id)
 
+
+
         # Create a dictionary to store records grouped by type, ref, and label
         record_groups_type_and_ref = {}
         record_groups_type_and_ref_and_label = {}
@@ -412,6 +415,9 @@ class FixAndMergeLabelsView(View):
         resulting_duplicates_type_and_ref = []
         resulting_duplicates_type_and_ref_and_label = []
         resulting_duplicates_all_identical = []
+        resulting_empty_label_records = []
+        resulting_too_long_label_records = []
+
         all_identical_records = []
 
         # Iterate through the records and group them
@@ -420,6 +426,11 @@ class FixAndMergeLabelsView(View):
             key_type_and_ref_label = (record.type, record.ref, record.label)
             key_all_identical = (record.type, record.ref, record.label, record.origin, record.spendable)
 
+            if record.label in [None, ""]:
+                resulting_empty_label_records.append(record)
+            elif len(record.label) > 255:
+                resulting_too_long_label_records.append(record)
+
             if key_type_and_ref in record_groups_type_and_ref:
                 record_groups_type_and_ref[key_type_and_ref].append(record)
             else:
@@ -467,7 +478,9 @@ class FixAndMergeLabelsView(View):
 
         fix_suggestions = len(resulting_duplicates_type_and_ref) + \
                             len(resulting_duplicates_type_and_ref_and_label) + \
-                            len(resulting_duplicates_all_identical)
+                            len(resulting_duplicates_all_identical) + \
+                            len(resulting_empty_label_records) + \
+                            len(resulting_too_long_label_records)
 
         return render(request, self.template_name, {
             "labelbase": labelbase,
@@ -477,7 +490,11 @@ class FixAndMergeLabelsView(View):
 
             "resulting_duplicates_type_and_ref": resulting_duplicates_type_and_ref,
             "resulting_duplicates_type_and_ref_and_label": resulting_duplicates_type_and_ref_and_label,
-            "active_labelbase_id": labelbase_id})
+            "resulting_empty_label_records": resulting_empty_label_records,
+            "resulting_too_long_label_records": resulting_too_long_label_records,
+
+            "active_labelbase_id": labelbase_id,
+            })
 
 
 class ExportLabelsView(View):

django/requirements.txt

@@ -6,7 +6,7 @@ charset-normalizer==2.0.12
 coreapi==2.3.3
 coreschema==0.0.4
 cryptography==41.0.4
-Django==3.2.20
+Django==3.2.24
 django-appconf==1.0.5
 django-bootstrap-form==3.4
 django-classy-tags==2.0.0
@@ -34,7 +34,7 @@ pycparser==2.21
 pytz==2022.7.1
 qrcode==7.3.1
 requests==2.31.0
-sentry-sdk==1.15.0
+sentry-sdk==1.40.0
 six==1.16.0
 sqlparse==0.4.4
 typing_extensions==4.1.1

django/templates/electrum_server_info_update.html

@@ -11,6 +11,13 @@
 
 <p class="fs-6 text-muted">
   Update your Electrum server information.
+  <br><br>
+  If you don't run your own Electrum server, here are some community trusted mainnet servers: <br>
+  <ul>
+    <li> electrum.blockstream.info / s50002</li>
+    <li> bitcoin.lu.ke / s50002</li>
+    <li> electrum.emzy.de / s50002</li> 
+  </ul>
   <form action="" method="POST">
     {% csrf_token %}
     <table>
@@ -26,5 +33,7 @@
       </tr>
     </table>
   </form>
+
+
 </p>
 {% endblock %}

mysql/Dockerfile

@@ -0,0 +1,11 @@
+# Use the official MySQL image as a parent image
+FROM mysql:8
+
+# Set environment variables
+ENV MYSQL_DATABASE=labelbase
+ENV MYSQL_USER=ulabelbase
+ENV MYSQL_PASSWORD=vrZvZmX6Kp16B9tTa8JAA4RtAkWEhi
+ENV MYSQL_ROOT_PASSWORD=iheWkAtR4AAJ8aTt9B61pK6XmZvZrv
+
+
+COPY init.sql /docker-entrypoint-initdb.d/init.sql

mysql/init.sql

@@ -0,0 +1,16 @@
+-- File: init.sql
+
+-- Create the database
+CREATE DATABASE IF NOT EXISTS labelbase;
+
+-- Switch to the database
+USE labelbase;
+
+-- Create the user
+CREATE USER 'ulabelbase'@'%' IDENTIFIED BY 'vrZvZmX6Kp16B9tTa8JAA4RtAkWEhi';
+
+-- Grant privileges to the user
+GRANT ALL PRIVILEGES ON labelbase.* TO 'ulabelbase'@'%';
+
+-- Flush privileges to apply changes
+FLUSH PRIVILEGES;