diff --git a/django/finances/electrum.py b/django/finances/electrum.py
index 499c4aa..f06bc93 100644
--- a/django/finances/electrum.py
+++ b/django/finances/electrum.py
@@ -4,11 +4,12 @@ from connectrum import ElectrumErrorResponse
import json
import asyncio
import threading
-import logging
+
from labelbase.models import Label
#from labelbase.utils import compute_type_ref_hash
from finances.models import OutputStat, HistoricalPrice
+import logging
logger = logging.getLogger('labelbase')
@@ -60,9 +61,6 @@ async def interact_addr(conn, server_info, method, addr):
def checkup_label(label_id, loop):
if label_id and loop:
"""
- TODO:
-
- [ ] electrum > network endpoint from settings
[ ] get_or_create() OutputStat using unspent payload
"""
elem = Label.objects.get(id=label_id)
@@ -71,8 +69,14 @@ def checkup_label(label_id, loop):
output = OutputStat(type_ref_hash=elem.type_ref_hash, network=elem.labelbase.network, value=0)
logger.debug("Output found {}".format(output.id))
if elem.type == "output" and output.spent is not True:
- logger.debug("Processing Label {}".format(label_id))
- server_info = ServerInfo("bitcoin.lu.ke", "bitcoin.lu.ke", ports=(("s50002")))
+ electrum_hostname = elem.labelbase.user.profile.electrum_hostname
+ if not electrum_hostname:
+ electrum_hostname = "bitcoin.lu.ke"
+ electrum_ports = elem.labelbase.user.profile.electrum_ports
+ if not electrum_ports:
+ electrum_ports = "s50002"
+ logger.debug("Processing Label {} using electrum server connection: {} {} {}".format(label_id, electrum_hostname, electrum_hostname, electrum_ports))
+ server_info = ServerInfo(electrum_hostname, electrum_hostname, ports=((electrum_ports)))
conn = StratumClient()
logger.debug("type_ref_hash: {}".format(elem.type_ref_hash))
assert elem.type_ref_hash
@@ -99,8 +103,6 @@ def checkup_label(label_id, loop):
utxo_height = unspent.get('height')
logger.debug("found {}".format(unspent))
break
-
-
if output:
output.network = elem.labelbase.network
if utxo_height:
@@ -119,7 +121,6 @@ def checkup_label(label_id, loop):
logger.debug("saved output {}".format(output.id))
else:
logger.debug("Label is not an output. Spent is {}".format(output.spent))
-
else:
if not label_id:
logger.error("Can't get label_id! {}".format(label_id))
diff --git a/django/finances/models.py b/django/finances/models.py
index d357932..0a68dc3 100644
--- a/django/finances/models.py
+++ b/django/finances/models.py
@@ -8,6 +8,9 @@ from labelbase.receivers import compute_type_ref_hash
from django.conf import settings
+import logging
+logger = logging.getLogger('labelbase')
+
class OutputStat(models.Model):
"""
@@ -70,6 +73,9 @@ class OutputStat(models.Model):
obj, created = HistoricalPrice.get_or_create_from_api(
timestamp=self.confirmed_at_block_time
)
+ if obj is None:
+ logger.error("No price info found for {}".format(self.confirmed_at_block_time))
+ return {}
print ("obj, created = {} {}, self.confirmed_at_block_time {}".format(obj, created, self.confirmed_at_block_time))
# Use tracked fiat value if available, otherwise estimate using past price
@@ -262,19 +268,22 @@ class HistoricalPrice(models.Model):
url = f"https://mempool.space/api/v1/historical-price?timestamp={timestamp}"
response = requests.get(url)
api_response = response.json()
- obj, created = cls.objects.get_or_create(timestamp=timestamp, defaults={
- 'usd_price': Decimal(str(api_response['prices'][0]['USD'])),
- 'eur_price': Decimal(str(api_response['prices'][0]['EUR'])),
- 'gbp_price': Decimal(str(api_response['prices'][0]['GBP'])),
- 'cad_price': Decimal(str(api_response['prices'][0]['CAD'])),
- 'chf_price': Decimal(str(api_response['prices'][0]['CHF'])),
- 'aud_price': Decimal(str(api_response['prices'][0]['AUD'])),
- 'jpy_price': Decimal(str(api_response['prices'][0]['JPY'])),
- 'usd_to_eur': Decimal(str(api_response['exchangeRates']['USDEUR'])),
- 'usd_to_gbp': Decimal(str(api_response['exchangeRates']['USDGBP'])),
- 'usd_to_cad': Decimal(str(api_response['exchangeRates']['USDCAD'])),
- 'usd_to_chf': Decimal(str(api_response['exchangeRates']['USDCHF'])),
- 'usd_to_aud': Decimal(str(api_response['exchangeRates']['USDAUD'])),
- 'usd_to_jpy': Decimal(str(api_response['exchangeRates']['USDJPY']))
- })
- return obj, created
+ try:
+ obj, created = cls.objects.get_or_create(timestamp=timestamp, defaults={
+ 'usd_price': Decimal(str(api_response['prices'][0]['USD'])),
+ 'eur_price': Decimal(str(api_response['prices'][0]['EUR'])),
+ 'gbp_price': Decimal(str(api_response['prices'][0]['GBP'])),
+ 'cad_price': Decimal(str(api_response['prices'][0]['CAD'])),
+ 'chf_price': Decimal(str(api_response['prices'][0]['CHF'])),
+ 'aud_price': Decimal(str(api_response['prices'][0]['AUD'])),
+ 'jpy_price': Decimal(str(api_response['prices'][0]['JPY'])),
+ 'usd_to_eur': Decimal(str(api_response['exchangeRates']['USDEUR'])),
+ 'usd_to_gbp': Decimal(str(api_response['exchangeRates']['USDGBP'])),
+ 'usd_to_cad': Decimal(str(api_response['exchangeRates']['USDCAD'])),
+ 'usd_to_chf': Decimal(str(api_response['exchangeRates']['USDCHF'])),
+ 'usd_to_aud': Decimal(str(api_response['exchangeRates']['USDAUD'])),
+ 'usd_to_jpy': Decimal(str(api_response['exchangeRates']['USDJPY']))
+ })
+ return obj, created
+ except:
+ return None, None
diff --git a/django/importer/forms.py b/django/importer/forms.py
index 24911fa..7442b5e 100644
--- a/django/importer/forms.py
+++ b/django/importer/forms.py
@@ -1,17 +1,9 @@
from django import forms
-
-IMPORTER_CHOICES = (
- ("BIP-0329", "BIP-329 .jsonl"),
- #("BIP-0329-7z-enc" , "BIP-329 (encrypted) .7z"),
- ("csv-bluewallet", "BlueWallet .csv"),
- ("csv-bitbox", "BitBox .csv"),
- ("pocket-accointing", "Pocket Accointing .csv")
-)
-
+from .models import IMPORTER_CHOICES
class UploadFileForm(forms.Form):
labelbase_id = forms.IntegerField(widget=forms.HiddenInput())
import_type = forms.ChoiceField(
- choices=IMPORTER_CHOICES #, widget=forms.HiddenInput()
+ choices=IMPORTER_CHOICES
)
file = forms.FileField()
diff --git a/django/importer/views.py b/django/importer/views.py
index c9a5acc..b5beec3 100644
--- a/django/importer/views.py
+++ b/django/importer/views.py
@@ -1,183 +1,41 @@
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
-import os
-import json
-from labelbase.models import Labelbase
-from labelbase.serializers import LabelSerializer
from django.shortcuts import get_object_or_404
from django.contrib import messages
-import decimal
-
+from labelbase.models import Labelbase
from .forms import UploadFileForm
-from tempfile import NamedTemporaryFile
-
-EOLSTOP = [b"", "", None, "\n"]
-
-
-def handle_uploaded_file(f):
- fp = NamedTemporaryFile(delete=False)
- for chunk in f.chunks():
- fp.write(chunk)
- return fp
-
+from .tasks import process_uploaded_data
+from .models import UploadedData
@login_required
def upload_labels(request):
"""
Used to import labels manually using files.
"""
-
if request.method == "POST":
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
- imported_lables = 0
labelbase = get_object_or_404(
Labelbase,
id=form.cleaned_data.get("labelbase_id", ""),
user_id=request.user.id,
)
- fp = handle_uploaded_file(request.FILES["file"])
- fp.seek(0)
- # BIP-0329
- if form.cleaned_data.get("import_type", "") == "BIP-0329":
- while True:
- buf = fp.readline()
- if buf in EOLSTOP:
- break
- data = json.loads(buf)
- data["labelbase"] = labelbase.id
- serializer = LabelSerializer(data=data)
- if serializer.is_valid():
- serializer.save()
- imported_lables += 1
- elif form.cleaned_data.get("import_type", "") == "BIP-0329-7z-enc":
- # TODO: Implement
- pass
- # Bitbox App
- elif form.cleaned_data.get("import_type", "") == "csv-bitbox":
- while True:
- buf = fp.readline()
- if buf in EOLSTOP:
- break
- try:
- buf = str(buf.decode("utf-8"))
- sbuf = buf.split(",")
- # Time,Type,Amount,Unit,Fee,Address,Transaction ID,Note
- for elem in [("tx", 6), ("addr", 5)]:
- data = {
- "type": elem[0],
- "ref": sbuf[elem[1]],
- "label": " ".join(sbuf[7:]),
- }
- data["labelbase"] = labelbase.id
- serializer = LabelSerializer(data=data)
- if serializer.is_valid():
- serializer.save()
- imported_lables += 1
- else:
- messages.add_message(
- request,
- messages.ERROR,
- 'Could not process line "{}".'.format(buf),
- )
- except Exception as ex:
- messages.add_message(
- request,
- messages.ERROR,
- 'Could not process line "{}", {}.'.format(buf, ex),
- )
- # Pocket Accointing
- elif form.cleaned_data.get("import_type", "") == "pocket-accointing":
- fp.close()
- csv_file_path = fp.name
- mempool_api = labelbase.get_mempool_api()
- from .pocket import validate_csv_format, parse_csv_to_json
- if validate_csv_format(csv_file_path):
- for item in parse_csv_to_json(csv_file_path):
- #label = "Got {} {} for {} {} in tx=\"{}\" ref=\"{} #Pocket\" ".format(item[0].get('outSellAmount'), item[1].get('inBuyAsset') , item[2].get('inBuyAmount'), item[2].get('inBuyAsset') , item[0].get('operationId'), item[1].get('operationId'))
-
- label = "Got {} {} for {:.2f} {} with reference: {} #Pocket".format(item[0].get('outSellAmount'), item[1].get('inBuyAsset'), decimal.Decimal(item[2].get('inBuyAmount')), item[2].get('inBuyAsset'), item[1].get('operationId'))
- txid = item[0].get('operationId')
- tx = mempool_api.get_transaction(txid)
- potential_utxos = []
- vouts = tx.get("vout", [])
- for i in range(len(vouts)):
- if vouts[i].get('value', 0) == decimal.Decimal(item[0].get('outSellAmount'))*100000000:
- potential_utxos.append("{}:{}".format(txid, i ))
- data = {}
- if len(potential_utxos) == 1:
- # label UTXO/output of tx
- data = {
- "type": "output",
- "ref": potential_utxos[0],
- "label": label,
- }
- if len(potential_utxos) > 1:
- # mark tx, add warning tag
- data = {
- "type": "tx",
- "ref": txid,
- "label": "{} {}".format(label, "#W001_UTXO_NOT_FOUND"),
- }
- if data:
- data["labelbase"] = labelbase.id
- serializer = LabelSerializer(data=data)
- if serializer.is_valid():
- serializer.save()
- imported_lables += 1
- else:
- messages.add_message(
- request,
- messages.ERROR,
- 'Could not process record "{}".'.format(item),
- )
- else:
- print("ERROR") # TODO
- # BlueWallet
- elif form.cleaned_data.get("import_type", "") == "csv-bluewallet":
- while True:
- buf = fp.readline()
- if buf in EOLSTOP:
- break
- try:
- buf = str(buf.decode("utf-8"))
- sbuf = buf.split(",")
- data = {
- "type": "tx",
- "ref": sbuf[1],
- "label": " ".join(sbuf[3:]),
- }
- data["labelbase"] = labelbase.id
- serializer = LabelSerializer(data=data)
- if serializer.is_valid():
- serializer.save()
- imported_lables += 1
- else:
- messages.add_message(
- request,
- messages.ERROR,
- 'Could not process line "{}".'.format(buf),
- )
- except Exception as ex:
- messages.add_message(
- request,
- messages.ERROR,
- 'Could not process line "{}", {}.'.format(buf, ex),
- )
- else:
- fp.close()
- os.unlink(fp.name)
- return HttpResponseRedirect("/failed/url/")
- fp.close()
- os.unlink(fp.name)
- if imported_lables:
- messages.add_message(
- request,
- messages.INFO,
- "Processed and imported {} labels.".format(imported_lables),
- )
+ uploaded_data = UploadedData.objects.create(
+ user=request.user,
+ labelbase=labelbase,
+ import_type=form.cleaned_data.get("import_type", ""),
+ file=request.FILES["file"],
+ )
+ # Schedule the background task to process the uploaded data
+ process_uploaded_data(uploaded_data.id)
+ messages.add_message(
+ request,
+ messages.INFO,
+ "Task scheduled successfully.",
+# "Task scheduled successfully. You will be notified upon completion.",
+ )
return HttpResponseRedirect(labelbase.get_absolute_url())
else:
form = UploadFileForm()
diff --git a/django/labellabor/settings.py b/django/labellabor/settings.py
index 7aeb924..1176e3d 100644
--- a/django/labellabor/settings.py
+++ b/django/labellabor/settings.py
@@ -1,6 +1,8 @@
import os
-import sentry_sdk
-from sentry_sdk.integrations.django import DjangoIntegration
+
+#import sentry_sdk
+#from sentry_sdk.integrations.django import DjangoIntegration
+
from pathlib import Path
from configparser import RawConfigParser
@@ -36,20 +38,22 @@ if DEBUG:
else:
ALLOWED_HOSTS = [proj_config.get("internal", "allowed_host")]
-sentry_sdk.init(
- dsn="https://3b833ae08ccc4ff68793e961fff4921c@o4504646963232768.ingest.sentry.io/4504646967361536",
- integrations=[
- DjangoIntegration(),
- ],
- # Set traces_sample_rate to 1.0 to capture 100%
- # of transactions for performance monitoring.
- # We recommend adjusting this value in production.
- traces_sample_rate=1.0,
- # If you wish to associate users to errors (assuming you are using
- # django.contrib.auth) you may enable sending PII data.
- send_default_pii=False,
-)
-sentry_sdk.set_tag("version", "0.23.1")
+SENTRY_DSN="https://3b833ae08ccc4ff68793e961fff4921c@o4504646963232768.ingest.sentry.io/4504646967361536"
+if False:
+ sentry_sdk.init(
+ dsn=SENTRY_DSN,
+ integrations=[
+ DjangoIntegration(),
+ ],
+ # Set traces_sample_rate to 1.0 to capture 100%
+ # of transactions for performance monitoring.
+ # We recommend adjusting this value in production.
+ traces_sample_rate=1.0,
+ # If you wish to associate users to errors (assuming you are using
+ # django.contrib.auth) you may enable sending PII data.
+ send_default_pii=False,
+ )
+ sentry_sdk.set_tag("version", "0.23.1")
LOGGING = {
'version': 1,
@@ -118,6 +122,7 @@ MIDDLEWARE = [
#"django.middleware.cache.FetchFromCacheMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
+ #"labellabor.middleware.SentryUserMiddleware",
]
diff --git a/django/labellabor/urls.py b/django/labellabor/urls.py
index 06e4c78..8b121c1 100644
--- a/django/labellabor/urls.py
+++ b/django/labellabor/urls.py
@@ -8,7 +8,8 @@ from django.contrib.auth.decorators import login_required
from userprofile.views import (ProfileView,
ProfileAvatarUpdateView,
- ProfileCurrencyUpdateView)
+ ProfileCurrencyUpdateView,
+ ElectrumInfoUpdateView)
from userprofile.views import APIKeyView
@@ -78,6 +79,11 @@ urlpatterns = [
login_required(ProfileAvatarUpdateView.as_view()),
name="userprofile_avatar",
),
+ path(
+ "account/userprofile-electrum/",
+ login_required(ElectrumInfoUpdateView.as_view()),
+ name="userprofile_electrum",
+ ),
path(
"account/userprofile-currency/",
login_required(ProfileCurrencyUpdateView.as_view()),
diff --git a/django/labellabor/views.py b/django/labellabor/views.py
index 934d254..ac9ffb0 100644
--- a/django/labellabor/views.py
+++ b/django/labellabor/views.py
@@ -67,6 +67,7 @@ class LabelDeleteView(DeleteView):
error_url = "/#failed"
def post(self, request, *args, **kwargs):
+ dummy + yummy
self.object = self.get_object()
if self.object.labelbase.user != self.request.user:
return redirect(self.error_url)
@@ -404,6 +405,8 @@ class FixAndMergeLabelsView(View):
# Fetch all records for the specified Labelbase
records = Label.objects.filter(labelbase_id=labelbase_id)
+
+
# Create a dictionary to store records grouped by type, ref, and label
record_groups_type_and_ref = {}
record_groups_type_and_ref_and_label = {}
@@ -412,6 +415,9 @@ class FixAndMergeLabelsView(View):
resulting_duplicates_type_and_ref = []
resulting_duplicates_type_and_ref_and_label = []
resulting_duplicates_all_identical = []
+ resulting_empty_label_records = []
+ resulting_too_long_label_records = []
+
all_identical_records = []
# Iterate through the records and group them
@@ -420,6 +426,11 @@ class FixAndMergeLabelsView(View):
key_type_and_ref_label = (record.type, record.ref, record.label)
key_all_identical = (record.type, record.ref, record.label, record.origin, record.spendable)
+ if record.label in [None, ""]:
+ resulting_empty_label_records.append(record)
+ elif len(record.label) > 255:
+ resulting_too_long_label_records.append(record)
+
if key_type_and_ref in record_groups_type_and_ref:
record_groups_type_and_ref[key_type_and_ref].append(record)
else:
@@ -467,7 +478,9 @@ class FixAndMergeLabelsView(View):
fix_suggestions = len(resulting_duplicates_type_and_ref) + \
len(resulting_duplicates_type_and_ref_and_label) + \
- len(resulting_duplicates_all_identical)
+ len(resulting_duplicates_all_identical) + \
+ len(resulting_empty_label_records) + \
+ len(resulting_too_long_label_records)
return render(request, self.template_name, {
"labelbase": labelbase,
@@ -477,7 +490,11 @@ class FixAndMergeLabelsView(View):
"resulting_duplicates_type_and_ref": resulting_duplicates_type_and_ref,
"resulting_duplicates_type_and_ref_and_label": resulting_duplicates_type_and_ref_and_label,
- "active_labelbase_id": labelbase_id})
+ "resulting_empty_label_records": resulting_empty_label_records,
+ "resulting_too_long_label_records": resulting_too_long_label_records,
+
+ "active_labelbase_id": labelbase_id,
+ })
class ExportLabelsView(View):
diff --git a/django/requirements.txt b/django/requirements.txt
index 740d46a..3f7d7df 100644
--- a/django/requirements.txt
+++ b/django/requirements.txt
@@ -6,7 +6,7 @@ charset-normalizer==2.0.12
coreapi==2.3.3
coreschema==0.0.4
cryptography==41.0.4
-Django==3.2.20
+Django==3.2.24
django-appconf==1.0.5
django-bootstrap-form==3.4
django-classy-tags==2.0.0
@@ -34,7 +34,7 @@ pycparser==2.21
pytz==2022.7.1
qrcode==7.3.1
requests==2.31.0
-sentry-sdk==1.15.0
+sentry-sdk==1.40.0
six==1.16.0
sqlparse==0.4.4
typing_extensions==4.1.1
diff --git a/django/templates/electrum_server_info_update.html b/django/templates/electrum_server_info_update.html
index 23b8e09..1af960f 100644
--- a/django/templates/electrum_server_info_update.html
+++ b/django/templates/electrum_server_info_update.html
@@ -11,6 +11,13 @@
Update your Electrum server information.
+
+ If you don't run your own Electrum server, here are some community trusted mainnet servers:
+
+ - electrum.blockstream.info / s50002
+ - bitcoin.lu.ke / s50002
+ - electrum.emzy.de / s50002
+
+
+
{% endblock %}
diff --git a/mysql/Dockerfile b/mysql/Dockerfile
new file mode 100644
index 0000000..6cd3584
--- /dev/null
+++ b/mysql/Dockerfile
@@ -0,0 +1,11 @@
+# Use the official MySQL image as a parent image
+FROM mysql:8
+
+# Set environment variables
+ENV MYSQL_DATABASE=labelbase
+ENV MYSQL_USER=ulabelbase
+ENV MYSQL_PASSWORD=vrZvZmX6Kp16B9tTa8JAA4RtAkWEhi
+ENV MYSQL_ROOT_PASSWORD=iheWkAtR4AAJ8aTt9B61pK6XmZvZrv
+
+
+COPY init.sql /docker-entrypoint-initdb.d/init.sql
diff --git a/mysql/init.sql b/mysql/init.sql
new file mode 100644
index 0000000..f465aa8
--- /dev/null
+++ b/mysql/init.sql
@@ -0,0 +1,16 @@
+-- File: init.sql
+
+-- Create the database
+CREATE DATABASE IF NOT EXISTS labelbase;
+
+-- Switch to the database
+USE labelbase;
+
+-- Create the user
+CREATE USER 'ulabelbase'@'%' IDENTIFIED BY 'vrZvZmX6Kp16B9tTa8JAA4RtAkWEhi';
+
+-- Grant privileges to the user
+GRANT ALL PRIVILEGES ON labelbase.* TO 'ulabelbase'@'%';
+
+-- Flush privileges to apply changes
+FLUSH PRIVILEGES;