diff --git a/.gitignore b/.gitignore index fd3bc6d..f8075e1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ db.sqlite3 config.ini labelbase.log +labelbase.log.* bgt.log db/ *.pyc diff --git a/django/background_task/management/commands/process_tasks.py b/django/background_task/management/commands/process_tasks.py index d0b7693..decf67d 100644 --- a/django/background_task/management/commands/process_tasks.py +++ b/django/background_task/management/commands/process_tasks.py @@ -1,11 +1,11 @@ import random import time +from django.db import close_old_connections from django.core.management.base import BaseCommand import asyncio from background_task.tasks import tasks, autodiscover from background_task.utils import SignalManager -from django.db import close_old_connections from background_task.recovery import start_recovery import logging diff --git a/django/background_task/management/commands/remove_completed.py b/django/background_task/management/commands/remove_completed.py new file mode 100644 index 0000000..923646d --- /dev/null +++ b/django/background_task/management/commands/remove_completed.py @@ -0,0 +1,20 @@ +from django.core.management.base import BaseCommand +from django.utils import timezone +from datetime import timedelta +import logging + +from background_task.models import CompletedTask + +logger = logging.getLogger('labelbase') + +class Command(BaseCommand): + help = "Remove completed tasks after 1 day." + + def handle(self, *args, **options): + try: + threshold = timezone.now() - timedelta(days=1) + deleted_count, _ = CompletedTask.objects.filter(locked_at__lte=threshold).delete() + self.stdout.write(self.style.SUCCESS(f"Deleted {deleted_count} completed tasks.")) + except Exception as ex: + logger.exception("Error occurred while deleting completed tasks") + self.stdout.write(self.style.ERROR("An error occurred while deleting completed tasks.")) diff --git a/django/finances/electrum.py b/django/finances/electrum.py index 20f6ac9..f177e26 100644 --- a/django/finances/electrum.py +++ b/django/finances/electrum.py @@ -57,6 +57,79 @@ def is_valid_output_ref(ref): return False def checkup_label(label_id, loop): + if label_id and loop: + try: + elem = Label.objects.get(id=label_id) + output = OutputStat.objects.filter(user=elem.labelbase.user, + type_ref_hash=elem.type_ref_hash, + network=elem.labelbase.network).last() + if not output: + output = OutputStat(user=elem.labelbase.user, + type_ref_hash=elem.type_ref_hash, + network=elem.labelbase.network, value=0) + + if elem.type == "output" and is_valid_output_ref(elem.ref) and \ + (output.spent is not True or output.confirmed_at_block_time == 0): + electrum_hostname = elem.labelbase.user.profile.electrum_hostname or "electrum.emzy.de" + electrum_ports = elem.labelbase.user.profile.electrum_ports or "s50002" + server_info = ServerInfo(electrum_hostname, electrum_hostname, ports=(electrum_ports)) + + conn = StratumClient() + utxo = elem.ref + utxo_resp = loop.run_until_complete(interact(conn, server_info, "blockchain.transaction.get", utxo)) + + if utxo_resp: + txid, index, address, value, blocktime = utxo_resp + if blocktime: + HistoricalPrice.get_or_create_from_api(timestamp=blocktime) + + unspents = loop.run_until_complete(interact_addr(conn, server_info, "blockchain.address.listunspent", address)) + + + utxo_value = 0 + utxo_height = 0 + + if unspents: + for unspent in unspents: + if unspent.get('tx_hash') == txid and \ + unspent.get('tx_pos') == int(index) and \ + unspent.get('height') > 0 and \ + unspent.get('value') > 0: # Output is confirmed, but not spent yet + output.spent = False + utxo_value = unspent.get('value') + utxo_height = unspent.get('height') + + output.network = elem.labelbase.network + if utxo_height: + output.confirmed_at_block_height = utxo_height + if blocktime: + output.confirmed_at_block_time = blocktime + if utxo_value: + output.value = utxo_value + elif value: + output.value = value + break + # + elif conn.last_error: + output.last_error = conn.last_error + else: + output.last_error = {} + output.save() + try: + conn.close() + except: + pass + + except Exception as e: + logger.error("Error processing label {}: {}".format(label_id, e)) + else: + if not label_id: + logger.error("Can't get label_id! {}".format(label_id)) + if not loop: + logger.error("Can't get loop!") + + +def checkup_label_buggy(label_id, loop): if label_id and loop: elem = Label.objects.get(id=label_id) output = OutputStat.objects.filter(user=elem.labelbase.user, @@ -69,7 +142,8 @@ def checkup_label(label_id, loop): network=elem.labelbase.network, value=0) print("Using OutputStat id {}".format(output)) print("elem.type {} {} {} {}".format(elem.type, is_valid_output_ref(elem.ref), elem.ref, output.spent)) - if elem.type == "output" and is_valid_output_ref(elem.ref) and output.spent is not True: + if elem.type == "output" and is_valid_output_ref(elem.ref) and \ + (output.spent is not True or output.confirmed_at_block_time == 0): electrum_hostname = elem.labelbase.user.profile.electrum_hostname if not electrum_hostname: electrum_hostname = "electrum.emzy.de" @@ -101,6 +175,7 @@ def checkup_label(label_id, loop): utxo_value = 0 utxo_height = 0 print("unspents: {}".format(unspents)) + if unspents: for unspent in unspents: if unspent.get('tx_hash') == tx_hash and \ diff --git a/django/finances/models.py b/django/finances/models.py index 282bfac..bc36104 100644 --- a/django/finances/models.py +++ b/django/finances/models.py @@ -14,6 +14,7 @@ import logging logger = logging.getLogger('labelbase') + class OutputStat(models.Model): """ These fields are unencrypted. Why? diff --git a/django/hashtags/__init__.py b/django/hashtags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django/hashtags/admin.py b/django/hashtags/admin.py new file mode 100644 index 0000000..ca52bf1 --- /dev/null +++ b/django/hashtags/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import Hashtag + +admin.site.register(Hashtag) diff --git a/django/hashtags/apps.py b/django/hashtags/apps.py new file mode 100644 index 0000000..a0e9b59 --- /dev/null +++ b/django/hashtags/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class HashtagsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'hashtags' diff --git a/django/hashtags/migrations/0001_initial.py b/django/hashtags/migrations/0001_initial.py new file mode 100644 index 0000000..e4a6271 --- /dev/null +++ b/django/hashtags/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.24 on 2024-03-12 10:43 + +from django.db import migrations, models +import django.db.models.deletion +import django_cryptography.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('labelbase', '0010_alter_label_label'), + ] + + operations = [ + migrations.CreateModel( + name='Hashtag', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', django_cryptography.fields.encrypt(models.CharField(blank=True, default='', max_length=160))), + ('description', django_cryptography.fields.encrypt(models.TextField(blank=True, default=''))), + ('labelbase', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labelbase.labelbase')), + ], + ), + ] diff --git a/django/hashtags/migrations/__init__.py b/django/hashtags/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django/hashtags/models.py b/django/hashtags/models.py new file mode 100644 index 0000000..ccf4490 --- /dev/null +++ b/django/hashtags/models.py @@ -0,0 +1,19 @@ +from django.db import models +from django.contrib.auth.models import User +from django_cryptography.fields import encrypt +from django.urls import reverse +from labelbase.models import Labelbase + +class Hashtag(models.Model): + labelbase = models.ForeignKey(Labelbase, on_delete=models.CASCADE) + name = encrypt( + models.CharField( + max_length=160, + default="", + blank=True, + ) + ) + description = encrypt(models.TextField(default="", blank=True)) + + def get_absolute_url(self): + return reverse('hashtag_edit', kwargs={'pk': self.pk}) diff --git a/django/hashtags/tasks.py b/django/hashtags/tasks.py new file mode 100644 index 0000000..6bf920d --- /dev/null +++ b/django/hashtags/tasks.py @@ -0,0 +1,20 @@ +import re +from .models import Hashtag +from labelbase.models import Label +from background_task import background + +@background(schedule=1) +def store_hashtags_as_objects(labelbase_id, loop=None): + for obj in Label.objects.filter(labelbase_id=labelbase_id): + hashtags = re.findall(r'#\w+', obj.label) + for tag in hashtags: + clean_tag = re.search(r'#(\w+)', tag).group(1) + hts = Hashtag.objects.filter(labelbase_id=labelbase_id) + found = False + for ht in hts: + if ht.name == clean_tag: + found = True + break + if not found: + hashtag = Hashtag(name=clean_tag, labelbase_id=labelbase_id) + hashtag.save() diff --git a/django/hashtags/views.py b/django/hashtags/views.py new file mode 100644 index 0000000..5a8fec3 --- /dev/null +++ b/django/hashtags/views.py @@ -0,0 +1,71 @@ +from django.views.generic import ListView +from django.views.generic.edit import UpdateView +from django.views import View +from django.http import HttpResponseForbidden, HttpResponseRedirect +from django.urls import reverse +from django.shortcuts import get_object_or_404 +from django.contrib import messages + +from labelbase.models import Labelbase +from .tasks import store_hashtags_as_objects +from .models import Hashtag + + +class LabelbaseProxyView(View): + def get(self, request, *args, **kwargs): + labelbase_id = kwargs.get('labelbase_id') + labelbase = get_object_or_404(Labelbase, id=labelbase_id, + user_id=request.user.id) + if request.GET.get('sync', 'no') == 'yes': + store_hashtags_as_objects(labelbase_id) + messages.add_message( + request, + messages.INFO, + "Processing Hashtags! Please wait briefly and refresh your page.", + ) + return HttpResponseRedirect(labelbase.get_hashtags_url()) + +class HashtagListView(ListView): + model = Hashtag + template_name = 'hashtags/labelbase_list.html' + + def get_queryset(self): + labelbase_id = self.kwargs['labelbase_id'] + labelbase = get_object_or_404( + Labelbase, id=labelbase_id, user_id=self.request.user.id + ) + queryset = Hashtag.objects.filter(labelbase_id=labelbase_id, + labelbase__user_id=self.request.user.id).order_by('-name') + return queryset + + def get_context_data(self, **kwargs): + context = super(HashtagListView, self).get_context_data(**kwargs) + context["active_labelbase_id"] = self.kwargs['labelbase_id'] + context["labelbase"] = get_object_or_404( + Labelbase, id=self.kwargs['labelbase_id'], user_id=self.request.user.id + ) + return context + + +class HashtagUpdateView(UpdateView): + model = Hashtag + fields = ['name', 'description'] + template_name = 'hashtags/labelbase_edit.html' + + def dispatch(self, request, *args, **kwargs): + if self.request.user.id != self.get_object().labelbase.user_id: + return HttpResponseForbidden("You do not have permission to edit this hashtag.") + return super().dispatch(request, *args, **kwargs) + + def get_success_url(self): + labelbase_id = self.object.labelbase_id + return reverse('labelbase_hashtags', kwargs={'labelbase_id': labelbase_id}) + + def get_context_data(self, **kwargs): + labelbase_id = self.get_object().labelbase.id + context = super(HashtagUpdateView, self).get_context_data(**kwargs) + context["active_labelbase_id"] = labelbase_id + context["labelbase"] = get_object_or_404( + Labelbase, id=labelbase_id, user_id=self.request.user.id + ) + return context diff --git a/django/knowledge_base/migrations/0004_auto_20240314_0939.py b/django/knowledge_base/migrations/0004_auto_20240314_0939.py new file mode 100644 index 0000000..8e3eb3f --- /dev/null +++ b/django/knowledge_base/migrations/0004_auto_20240314_0939.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.24 on 2024-03-14 09:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('knowledge_base', '0003_auto_20240308_2231'), + ] + + operations = [ + migrations.AddField( + model_name='article', + name='order', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='category', + name='desc', + field=models.CharField(default='', max_length=280), + ), + migrations.AddField( + model_name='category', + name='order', + field=models.IntegerField(default=0), + ), + ] diff --git a/django/knowledge_base/models.py b/django/knowledge_base/models.py index 90524b8..075c3bc 100644 --- a/django/knowledge_base/models.py +++ b/django/knowledge_base/models.py @@ -14,6 +14,8 @@ class ExportSnapshot(models.Model): class Category(models.Model): name = models.CharField(max_length=100) + desc = models.CharField(max_length=280, default="") + order = models.IntegerField(default=0) slug = models.SlugField(unique=True, max_length=100) exclude_from_export = models.BooleanField(default=False) @@ -27,6 +29,7 @@ class Category(models.Model): class Article(models.Model): title = models.CharField(max_length=200) content = models.TextField() + order = models.IntegerField(default=0) category = models.ForeignKey(Category, on_delete=models.CASCADE) slug = models.SlugField(unique=True, max_length=100) exclude_from_export = models.BooleanField(default=False) diff --git a/django/knowledge_base/urls.py b/django/knowledge_base/urls.py index 4179839..7725445 100644 --- a/django/knowledge_base/urls.py +++ b/django/knowledge_base/urls.py @@ -8,6 +8,5 @@ urlpatterns = [ path('article//', ArticleDetailView.as_view(), name='article_detail'), #TODO: staff only, path('export', ExportJSONView.as_view(), name='export_json'), - #TODO: staff only, path('import', ImportJSONView.as_view(), name='import_json'), ] diff --git a/django/knowledge_base/views.py b/django/knowledge_base/views.py index b2dd344..7ce24a3 100644 --- a/django/knowledge_base/views.py +++ b/django/knowledge_base/views.py @@ -13,7 +13,6 @@ class ExportJSONView(View): for category in categories: articles = list(category.article_set.all().exclude(exclude_from_export=True).values()) data[category.name] = articles - export_snapshot = ExportSnapshot.objects.create( exported_data=data ) @@ -37,7 +36,8 @@ class ImportJSONView(View): try: category = Category.objects.get(slug=slugify(category_name)) except Category.DoesNotExist: - category = Category.objects.create(name=category_name, slug=slugify(category_name)) + category = Category.objects.create(name=category_name, + slug=slugify(category_name)) for article_data in articles: try: @@ -47,8 +47,10 @@ class ImportJSONView(View): article.category = category article.save() except Article.DoesNotExist: - Article.objects.create(title=article_data['title'], content=article_data['content'], category=category, slug=article_data['slug']) - + Article.objects.create(title=article_data['title'], + content=article_data['content'], + category=category, + slug=article_data['slug']) return JsonResponse({'message': 'Import successful'}) except json.JSONDecodeError: return JsonResponse({'error': 'Invalid JSON format'}, status=400) diff --git a/django/labelbase/models.py b/django/labelbase/models.py index 0bbd6be..667d68f 100644 --- a/django/labelbase/models.py +++ b/django/labelbase/models.py @@ -108,6 +108,8 @@ class Labelbase(models.Model): def get_absolute_url(self): return reverse("labelbase", args=[self.id]) + def get_hashtags_url(self): + return reverse('labelbase_hashtags', kwargs={'labelbase_id': self.id}) class Label(models.Model): """ @@ -193,6 +195,7 @@ class Label(models.Model): """ return self.labelbase.get_absolute_url() + def get_mempool_url(self): if self.labelbase and self.labelbase.network != "mainnet": mempool_endpoint = \ diff --git a/django/labelbase/static/css/labelbase.css b/django/labelbase/static/css/labelbase.css index 683af45..30fa9a2 100644 --- a/django/labelbase/static/css/labelbase.css +++ b/django/labelbase/static/css/labelbase.css @@ -237,7 +237,7 @@ box-shadow: 0 0 0 3px rgba(255, 255, 255, .25); --bd-callout-border: var(--bs-warning-border-subtle); } - .bd-callout-good { + .bd-callout-good { --bd-callout-color: var(--bs-success-text-emphasis); --bd-callout-bg: #d9fbd0; --bd-callout-border: #bee8be; @@ -285,17 +285,10 @@ box-shadow: 0 0 0 3px rgba(255, 255, 255, .25); } - .modal-footer { - border-top: 0; - } - .modal-header { - border-bottom: 0; - } - -/* -*/ + .modal-header { + border-bottom: 0; + } diff --git a/django/labellabor/settings.py b/django/labellabor/settings.py index a6a1b53..6e76b17 100644 --- a/django/labellabor/settings.py +++ b/django/labellabor/settings.py @@ -109,6 +109,7 @@ INSTALLED_APPS = [ "background_task", "connectrum", "knowledge_base", + "hashtags", ] diff --git a/django/labellabor/urls.py b/django/labellabor/urls.py index 47c2118..7e409e5 100644 --- a/django/labellabor/urls.py +++ b/django/labellabor/urls.py @@ -12,6 +12,8 @@ from userprofile.views import (ProfileView, ElectrumInfoUpdateView) from userprofile.views import APIKeyView +from hashtags.views import HashtagListView, HashtagUpdateView, LabelbaseProxyView + from .views import ( LabelbaseView, @@ -27,7 +29,7 @@ from .views import ( RegistrationView, LabelbaseFormView, LabelbaseUpdateView, - FaqView, + DonationView, AboutView, EncryptionView, InteroperationalView, @@ -37,6 +39,7 @@ from .views import ( FixAndMergeLabelsView, LabelbaseDatatableView, LabelbasePortfolioView, + OutputStatUpdateRedirectView, ) from importer.views import upload_labels @@ -99,6 +102,21 @@ urlpatterns = [ login_required(LabelbaseView.as_view()), name="labelbase" ), + path( + "labelbase//hashtags/", + login_required(HashtagListView.as_view()), + name="labelbase_hashtags" + ), + path( + "labelbase//hashtags/proxy/", + login_required(LabelbaseProxyView.as_view()), + name="labelbase_hashtags_proxy" + ), + path( + "labelbase/hashtag//edit/", + login_required(HashtagUpdateView.as_view()), + name="hashtag_edit" + ), path( "labelbase//data/", login_required(LabelbaseDatatableView.as_view()), @@ -206,9 +224,9 @@ urlpatterns = [ name="terms" ), path( - "faq", - FaqView.as_view(), - name="faq" + "donate", + DonationView.as_view(), + name="donate" ), path( "about", @@ -225,6 +243,11 @@ urlpatterns = [ InteroperationalView.as_view(), name="interoperational" ), + path( + "outputstat//update//", + login_required(OutputStatUpdateRedirectView.as_view()), + name='outputstat_update_redirect' + ), path( "", HomeView.as_view(), diff --git a/django/labellabor/views.py b/django/labellabor/views.py index e54d62f..88eb6e2 100644 --- a/django/labellabor/views.py +++ b/django/labellabor/views.py @@ -12,24 +12,24 @@ from django.views.generic.list import ListView from django.views.generic.edit import DeleteView from django.views.generic.edit import UpdateView from django.shortcuts import get_object_or_404 +from django.http import HttpResponseRedirect +from django.http import FileResponse +from django.urls import reverse +from django.template.loader import render_to_string +from django.contrib import messages +from django_datatables_view.base_datatable_view import BaseDatatableView from two_factor.views import OTPRequiredMixin from two_factor.views.utils import class_view_decorator +from rest_framework.authtoken.models import Token +from bip329.bip329_writer import BIP329JSONLWriter, BIP329JSONLEncryptedWriter from labelbase.models import Label, Labelbase from labelbase.forms import LabelForm, LabelbaseForm from labelbase.forms import ExportLabelsForm -from django.http import HttpResponseRedirect -from django.http import FileResponse -from rest_framework.authtoken.models import Token -from bip329.bip329_writer import BIP329JSONLWriter, BIP329JSONLEncryptedWriter -from django.urls import reverse -from django_datatables_view.base_datatable_view import BaseDatatableView -from django.template.loader import render_to_string -from django.contrib import messages - - -from .utils import hashtag_to_badge, extract_fiat_value from finances.models import OutputStat from finances.tasks import check_all_outputs +from .utils import hashtag_to_badge, extract_fiat_value + + class AboutView(TemplateView): @@ -56,8 +56,8 @@ class TermsView(TemplateView): template_name = "terms.html" -class FaqView(TemplateView): - template_name = "faq.html" +class DonationView(TemplateView): + template_name = "donate.html" class LabelDeleteView(DeleteView): @@ -648,3 +648,49 @@ class RegistrationCompleteView(TemplateView): @class_view_decorator(never_cache) class ExampleSecretView(OTPRequiredMixin, TemplateView): template_name = "secret.html" + + +class OutputStatUpdateRedirectView(View): + def get(self, request, output_stats_id, label_id): + try: + output_stat = OutputStat.objects.get(id=output_stats_id) + except OutputStat.DoesNotExist: + messages.add_message( + request, + messages.ERROR, + "Hmmmm.... Could not modify label data." + ) + return redirect('edit_label', pk=label_id) + try: + label = Label.objects.get(id=label_id, labelbase__user_id=self.request.user.id) + except Label.DoesNotExist: + messages.add_message( + request, + messages.ERROR, + "Can't find what you are looking for.." + ) + return redirect("home") + + spent = request.GET.get('force-spent', None) + + if spent not in ["true", "false", "none"]: + messages.add_message( + request, + messages.ERROR, + "Hmmmm.... This action is unknown." + ) + return redirect('edit_label', pk=label_id) + if spent == "true": + spent = True + elif spent == "false": + spent = False + elif spent == "none": + spent = None + OutputStat.objects.filter(id=output_stat.id).update(spent=spent) + messages.add_message( + request, + messages.SUCCESS, + "Okay! Verifying output status now." + ) + label.save() # will trigger a check agains Electrum + return redirect('edit_label', pk=label_id) diff --git a/django/templates/_base.html b/django/templates/_base.html index 70236df..45c65cd 100644 --- a/django/templates/_base.html +++ b/django/templates/_base.html @@ -15,8 +15,11 @@ {% block extra_media %}{% endblock %} + + @@ -120,7 +123,7 @@
  • Fiat Finances
    • {% endif %} {% if labelbase.user.profile.use_hashtags %} -
  • Hashtags
    • +
  • Hashtags
    • {% endif %} @@ -178,15 +181,17 @@ Community - +
  • + Donate +
  • Newsletter
  • -
  • +
  • Code on GitHub @@ -205,7 +210,7 @@
  • -->
  • - FAQ & Knowledge + Knowledge Base
  • @@ -222,18 +227,10 @@ - {% comment %} - -
    - {% endcomment %} + -
    {% include "_messages.html" %} @@ -453,7 +450,7 @@ data: { toggleId: 'use_chatwoot', toggleValue: 'true' }, success: function (response) { show_message("success", "Success! " + response.message) - window.$chatwoot.toggle("open"); + window.$chatwoot.toggle("open"); }, error: function (xhr, status, error) { console.error(error); diff --git a/django/templates/_intro.html b/django/templates/_intro.html index 06c3f03..5279164 100644 --- a/django/templates/_intro.html +++ b/django/templates/_intro.html @@ -1,5 +1,7 @@ - + {% endaddtoblock %} + +