Merge pull request #29 from tpokorra/TP-202105-fees_in_crypto

Tp 202105 fees in crypto

Makefile

@@ -0,0 +1,16 @@
+VENV := . .venv/bin/activate &&
+
+create_venv:
+	python3 -m venv .venv
+
+create_db:
+	${VENV} python manage.py migrate
+	${VENV} echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.filter(is_superuser=True).exists() or User.objects.create_superuser('admin', 'admin@example.com', 'admin')" | python manage.py shell
+	if [ -f transactions.sql ]; then cat transactions.sql | sqlite3 db.sqlite3; fi
+	if [ -f exchangerates.sql ]; then cat exchangerates.sql | sqlite3 db.sqlite3; fi
+
+clean:
+	rm db.sqlite3
+
+run:
+	${VENV} python manage.py runserver 0.0.0.0:8000

apps/monitor/calc.py

@@ -64,7 +64,7 @@ class Calc:
     out["crypto"] = crypto
 
     with connection.cursor() as cursor:
-        sql = """SELECT SUM(amount_after_fee) as after_fee, SUM(amount) as amount FROM `transaction` WHERE amount_after_fee > 0 and crypto_currency=%s and owner_id=%s"""
+        sql = """SELECT SUM(crypto_amount) as crypto_amount, SUM(fiat_amount) as fiat_amount FROM `transaction` WHERE transaction_type = 'B' and crypto_currency=%s and owner_id=%s"""
         cursor.execute(sql, [crypto, userid])
         bought = cursor.fetchone()
         if bought[0]:
@@ -75,24 +75,24 @@ class Calc:
           out["bought"] = None
     
     with connection.cursor() as cursor:
-        sql = """SELECT SUM(amount_before_fee) as before_fee, SUM(amount) as amount FROM `transaction` WHERE amount_before_fee < 0 and crypto_currency=%s and owner_id=%s"""
+        sql = """SELECT SUM(crypto_amount) as crypto_amount, SUM(fiat_amount) as fiat_amount FROM `transaction` WHERE transaction_type = 'S' and crypto_currency=%s and owner_id=%s"""
         cursor.execute(sql, [crypto, userid])
         sold = cursor.fetchone()
         if sold[0]:
           out["sold"] = sold[1]
-          total_investment += Decimal(sold[1])
+          total_investment -= Decimal(sold[1])
-          amount_kept += Decimal(sold[0])
+          amount_kept -= Decimal(sold[0])
         else:
           out["sold"] = None
 
-    # moving coins to other wallet with fees.
+    # calculate all fees.
     with connection.cursor() as cursor:
-        sql = """SELECT SUM(amount_after_fee) as after_fee FROM `transaction` WHERE amount_before_fee = 0 and crypto_currency=%s and owner_id=%s"""
+        sql = """SELECT SUM(crypto_fee) as crypto_fee, SUM(fiat_fee) as fiat_fee FROM `transaction` WHERE crypto_currency=%s and owner_id=%s"""
         cursor.execute(sql, [crypto, userid])
-        sold = cursor.fetchone()
+        fees = cursor.fetchone()
-        if sold[0]:
+        if fees[0]:
-          # total_investment -= Decimal(sold[1])
+          total_investment -= Decimal(fees[1])
-          amount_kept += Decimal(sold[0])
+          amount_kept -= Decimal(fees[0])
 
     rateEUR = ExchangeRate.objects.filter(crypto_currency=crypto, fiat_currency='EUR').order_by('-datetime_valid').first()
     rateUSD = ExchangeRate.objects.filter(crypto_currency=crypto, fiat_currency='USD').order_by('-datetime_valid').first()
@@ -130,14 +130,17 @@ class Calc:
 
     amount_within_past_year = 0
     with connection.cursor() as cursor:
-        sql = """SELECT SUM(amount_after_fee) as amount FROM `transaction` WHERE amount > 0 and crypto_currency=%s and owner_id=%s and date_valid>%s"""
+        sql = """SELECT SUM(crypto_amount) as amount FROM `transaction` WHERE transaction_type='B' and crypto_currency=%s and owner_id=%s and date_valid>%s"""
         cursor.execute(sql, [crypto, userid, datetime.datetime.now() - datetime.timedelta(days=365)])
         bought = cursor.fetchone()
         out["bought_recently"] = None
         if bought[0]:
           out["bought_recently"] = bought[0]
           amount_within_past_year = Decimal(bought[0])
-        amount_available_to_sell = amount_kept - amount_within_past_year
+        if amount_kept > amount_within_past_year:
+          amount_available_to_sell = amount_kept - amount_within_past_year
+        else:
+          amount_available_to_sell = 0
         out["bought_tax_free"] = None
         out["value_tax_free"] = None
         if rateEUR:

apps/monitor/templates/monitor.html

@@ -64,7 +64,7 @@
               We have sold {{ c.crypto }} for {{ c.sold|floatformat:2 }} EUR<br/>
               {% endif %}
               {% if c.amount_kept %}
-              Current value of our {{ c.amount_kept|formatcurrency }} {{ c.crypto }} is {{ c.current_value|floatformat:2 }} EUR<br/>
+              Current value of our <span title="{{ c.amount_kept }} {{ c.crypto }}">{{ c.amount_kept|formatcurrency }} {{ c.crypto }}</span> is {{ c.current_value|floatformat:2 }} EUR<br/>
               {% endif %}
               {% if c.bought_recently %}
               We have bought within the last 12 months:  {{ c.bought_recently|formatcurrency }} {{ c.crypto }}<br/>

apps/monitor/templatetags/filter.py

@@ -4,6 +4,8 @@ register = template.Library()
 
 @register.filter
 def formatcurrency(value):
+    if not value:
+        return "0.00"
     if value > 100:
         return "%0.0f" % (value,)
     if value < 0.01:

apps/transactions/forms.py

@@ -1,4 +1,5 @@
 from django import forms  
+import sys
 import datetime
 from apps.transactions.models import Transaction
 from apps.rates.models import ExchangeRate
@@ -16,15 +17,16 @@ class TransactionForm(forms.ModelForm):
         model = Transaction
         fields = "__all__"
 
-        exchangerates = ExchangeRate.objects.filter(datetime_valid__gt = datetime.datetime.today() - datetime.timedelta(days=7))
+        # do not run this code in initial migration, because the database has not been built yet
-        crypto_currencies = [(x, x) for x in sorted({ex.crypto_currency for ex in exchangerates})]
+        if not "migrate" in sys.argv:
-        fiat_currencies = [(x, x) for x in sorted({ex.fiat_currency for ex in exchangerates})]
+            exchangerates = ExchangeRate.objects.filter(datetime_valid__gt = datetime.datetime.today() - datetime.timedelta(days=7))
-
+            crypto_currencies = [(x, x) for x in sorted({ex.crypto_currency for ex in exchangerates})]
-        widgets = {
+            fiat_currencies = [(x, x) for x in sorted({ex.fiat_currency for ex in exchangerates})]
-            'fiat_currency': forms.Select(choices = fiat_currencies),
-            'crypto_currency': forms.Select(choices = crypto_currencies),
-        }
 
+            widgets = {
+                'fiat_currency': forms.Select(choices = fiat_currencies),
+                'crypto_currency': forms.Select(choices = crypto_currencies),
+            }
 
 class ImportForm(forms.Form):
     api_key = forms.CharField(label='API Key', widget=forms.PasswordInput, max_length=100)

apps/transactions/importbtcde.py

@@ -18,7 +18,7 @@ class ImportBtcDe:
           break
       for trade in trades:
 
-        # TODO: check if this trade_id does not exist in the database for this user
+        # check if this trade_id does not exist in the database for this user
         with connection.cursor() as cursor:
           sql = """SELECT trade_id FROM `transaction` WHERE trade_id=%s and owner_id=%s"""
           cursor.execute(sql, [trade['trade_id'], Owner.id])
@@ -37,14 +37,18 @@ class ImportBtcDe:
           if trade['trading_pair'].upper().endswith(f):
             t.fiat_currency = f
         t.owner = Owner
-        t.amount_before_fee = Decimal(trade['amount'])
-        t.amount_after_fee = Decimal(trade['amount']) - Decimal(trade['fee_currency'])
-        t.amount = Decimal(trade['volume'])
         if trade['type'] == 'sell':
-            t.amount -= Decimal(trade['fee_eur'])
+            t.crypto_amount = Decimal(trade['amount_currency_to_trade'])-Decimal(trade['fee_currency_to_trade'])
-            t.amount_before_fee *= -1
+        else:
-            t.amount_after_fee *= -1
+            t.crypto_amount = Decimal(trade['amount_currency_to_trade'])
-            t.amount *= -1
+        t.crypto_fee = Decimal(trade['fee_currency_to_trade'])
+        t.fiat_amount = Decimal(trade['volume_currency_to_pay'])
+        t.fiat_fee = Decimal(trade['fee_currency_to_pay'])
+        if trade['type'] == 'sell':
+            t.transaction_type = 'S'
+        else:
+            t.transaction_type = 'B'
+        t.description = 'Bitcoin.de Trade'
         t.exchange_rate = trade['price']
         t.date_valid = trade['created_at']
         t.save()

apps/transactions/migrations/0003_auto_20210511_0703.py

@@ -0,0 +1,38 @@
+# Generated by Django 3.2.2 on 2021-05-11 05:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0002_transaction_trade_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='crypto_amount',
+            field=models.DecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='crypto_fee',
+            field=models.DecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='fiat_amount',
+            field=models.DecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='fiat_fee',
+            field=models.DecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='transaction_type',
+            field=models.CharField(choices=[('B', 'Buy Crypto'), ('S', 'Sell Crypto'), ('T', 'Transfer Crypto')], default='B', max_length=1),
+        ),
+    ]

apps/transactions/migrations/0004_auto_20210511_0703.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.2 on 2021-05-11 05:03
+
+from django.db import migrations
+
+def separate_fees(apps, schema_editor):
+
+    Transaction = apps.get_model('transactions', 'Transaction')
+    for tr in Transaction.objects.all():
+        if tr.amount_before_fee > 0:
+            tr.crypto_fee = tr.amount_before_fee - tr.amount_after_fee
+            tr.crypto_amount = tr.amount_before_fee
+            tr.transaction_type = 'B'
+        elif tr.amount_before_fee < 0:
+            tr.crypto_fee = -1 * tr.amount_before_fee + tr.amount_after_fee
+            tr.crypto_amount = -1 * tr.amount_before_fee
+            tr.transaction_type = 'S'
+        else:
+            tr.crypto_fee = 0
+            tr.crypto_amount = 0
+            tr.transaction_type = 'T'
+
+        tr.fiat_amount = tr.amount
+        if tr.fiat_amount < 0:
+            tr.fiat_amount = -1 * tr.fiat_amount
+        tr.fiat_fee = 0
+
+        tr.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0003_auto_20210511_0703'),
+    ]
+
+    operations = [
+        migrations.RunPython(separate_fees),
+    ]

apps/transactions/migrations/0005_auto_20210511_1638.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.2 on 2021-05-11 16:38
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0004_auto_20210511_0703'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='transaction',
+            name='amount',
+        ),
+        migrations.RemoveField(
+            model_name='transaction',
+            name='amount_after_fee',
+        ),
+        migrations.RemoveField(
+            model_name='transaction',
+            name='amount_before_fee',
+        ),
+    ]

apps/transactions/migrations/0006_auto_20210511_2230.py

@@ -0,0 +1,44 @@
+# Generated by Django 3.2.2 on 2021-05-11 20:30
+
+import apps.transactions.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0005_auto_20210511_1638'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='description',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transaction',
+            name='crypto_amount',
+            field=apps.transactions.models.NonscientificDecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transaction',
+            name='crypto_fee',
+            field=apps.transactions.models.NonscientificDecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transaction',
+            name='exchange_rate',
+            field=apps.transactions.models.NonscientificDecimalField(decimal_places=10, max_digits=24),
+        ),
+        migrations.AlterField(
+            model_name='transaction',
+            name='fiat_amount',
+            field=apps.transactions.models.NonscientificDecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+        migrations.AlterField(
+            model_name='transaction',
+            name='fiat_fee',
+            field=apps.transactions.models.NonscientificDecimalField(blank=True, decimal_places=10, max_digits=24, null=True),
+        ),
+    ]

apps/transactions/models.py

@@ -1,16 +1,46 @@
 from django.db import models
 from django.contrib.auth.models import User
+from decimal import Decimal, Context
+
+class NonscientificDecimalField(models.DecimalField):
+    """ Prevents values from being displayed with E notation, with trailing 0's
+        after the decimal place  truncated. (This causes precision to be lost in
+        many cases, but is more user friendly and consistent for non-scientist
+        users)
+    """
+    def value_from_object(self, obj):
+        def remove_exponent(val):
+            """Remove exponent and trailing zeros.
+               >>> remove_exponent(Decimal('5E+3'))
+               Decimal('5000')
+            """
+            context = Context(prec=self.max_digits)
+            return val.quantize(Decimal(1), context=context) if val == val.to_integral() else val.normalize(context)
+
+        val = super(NonscientificDecimalField, self).value_from_object(obj)
+        if isinstance(val, Decimal):
+            return remove_exponent(val)
 
 # Create your models here.
 class Transaction(models.Model):
+    TRANSACTION_TYPES = (
+        ('B', 'Buy Crypto'),
+        ('S', 'Sell Crypto'),
+        ('T', 'Transfer Crypto'),
+    )
+
     trade_id = models.CharField(max_length=20, default='MANUAL')
-    crypto_currency = models.CharField(max_length=10)
     owner = models.ForeignKey(User, on_delete=models.CASCADE)
-    amount_before_fee = models.DecimalField(max_digits=24, decimal_places=10)
+    crypto_currency = models.CharField(max_length=10)
-    amount_after_fee = models.DecimalField(max_digits=24, decimal_places=10)
+    crypto_amount = NonscientificDecimalField(max_digits=24, decimal_places=10, null=True, blank=True)
-    exchange_rate = models.DecimalField(max_digits=24, decimal_places=10)
+    crypto_fee = NonscientificDecimalField(max_digits=24, decimal_places=10, null=True, blank=True)
+    exchange_rate = NonscientificDecimalField(max_digits=24, decimal_places=10)
     fiat_currency = models.CharField(max_length=10)
-    amount = models.DecimalField(max_digits=24, decimal_places=10)
+    fiat_amount = NonscientificDecimalField(max_digits=24, decimal_places=10, null=True, blank=True)
+    fiat_fee = NonscientificDecimalField(max_digits=24, decimal_places=10, null=True, blank=True)
     date_valid = models.DateTimeField('date transfered')
+    transaction_type = models.CharField(max_length=1, choices=TRANSACTION_TYPES, default='B')
+    description = models.CharField(max_length=100, null=True, blank=True)
+
     class Meta:  
         db_table = "transaction"

apps/transactions/templates/add.html

@@ -28,6 +28,13 @@
 	</div>
 	</div>
 
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Transaction Type:</label>
+	<div class="col-sm-4">
+	  {{ form.transaction_type }}
+	</div>
+	</div>
+
 	<div class="form-group row">
 	<label class="col-sm-2 col-form-label">Trade ID:</label>
 	<div class="col-sm-4">
@@ -43,16 +50,16 @@
 	</div>
 
 	<div class="form-group row">
-	<label class="col-sm-2 col-form-label">Amount before fee:</label>
+	<label class="col-sm-2 col-form-label">Crypto Amount:</label>
 	<div class="col-sm-4">
-	  {{ form.amount_before_fee }}
+	  {{ form.crypto_amount }}
 	</div>
 	</div>
 
 	<div class="form-group row">
-	<label class="col-sm-2 col-form-label">Amount after fee:</label>
+	<label class="col-sm-2 col-form-label">Crypto fee:</label>
 	<div class="col-sm-4">
-	  {{ form.amount_after_fee }}
+	  {{ form.crypto_fee }}
 	</div>
 	</div>
 
@@ -73,7 +80,14 @@
 	<div class="form-group row">
 	<label class="col-sm-2 col-form-label">Fiat Amount:</label>
 	<div class="col-sm-4">
-	  {{ form.amount }}
+	  {{ form.fiat_amount }}
+	</div>
+	</div>
+
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Fiat Fee:</label>
+	<div class="col-sm-4">
+	  {{ form.fiat_fee }}
 	</div>
 	</div>
 
@@ -84,6 +98,13 @@
 	</div>
 	</div>
 
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Description:</label>
+	<div class="col-sm-4">
+	  {{ form.description }}
+	</div>
+	</div>
+
 	<div class="form-group row">
 	<label class="col-sm-1 col-form-label"></label>
 	<div class="col-sm-4">

apps/transactions/templates/edit.html

@@ -29,24 +29,31 @@
 	</div>
 	</div>
 
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Transaction Type:</label>
+	<div class="col-sm-4">
+	  {{ form.transaction_type }}
+	</div>
+	</div>
+
 	<div class="form-group row">
 	<label class="col-sm-2 col-form-label">Crypto Currency:</label>
 	<div class="col-sm-4">
-		{{ form.crypto_currency }}
+	  {{ form.crypto_currency }}
 	</div>
 	</div>
 
 	<div class="form-group row">
-	<label class="col-sm-2 col-form-label">Amount before fee:</label>
+	<label class="col-sm-2 col-form-label">Crypto Amount:</label>
 	<div class="col-sm-4">
-		{{ form.amount_before_fee }}
+	  {{ form.crypto_amount }}
 	</div>
 	</div>
 
 	<div class="form-group row">
-	<label class="col-sm-2 col-form-label">Amount after fee:</label>
+	<label class="col-sm-2 col-form-label">Crypto fee:</label>
 	<div class="col-sm-4">
-		{{ form.amount_after_fee }}
+	  {{ form.crypto_fee }}
 	</div>
 	</div>
 
@@ -67,14 +74,28 @@
 	<div class="form-group row">
 	<label class="col-sm-2 col-form-label">Fiat Amount:</label>
 	<div class="col-sm-4">
-		{{ form.amount }}
+	  {{ form.fiat_amount }}
+	</div>
+	</div>
+
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Fiat Fee:</label>
+	<div class="col-sm-4">
+	  {{ form.fiat_fee }}
 	</div>
 	</div>
 
 	<div class="form-group row">
 	<label class="col-sm-2 col-form-label">{{ form.date_valid.label }}:</label>
 	<div class="col-sm-4">
-		{{ form.date_valid }}
+	  {{ form.date_valid }}
+	</div>
+	</div>
+
+	<div class="form-group row">
+	<label class="col-sm-2 col-form-label">Description:</label>
+	<div class="col-sm-4">
+	  {{ form.description }}
 	</div>
 	</div>
 

apps/transactions/templates/show.html

@@ -20,15 +20,26 @@
 	</thead>
 	<tbody>
 {% for transaction in transactions %}
+    {% if transaction.description and transaction.description != '' %}
+    <tr>
+        <td colspan="5">{{ transaction.description }}:</td>
+    </tr>
+    {% endif %}
 	<tr>
 		<td>{{ transaction.date_valid }}</td>
-		<td>{{ transaction.amount_before_fee|floatformat:3 }} {{ transaction.crypto_currency }}</td>
+		<td>{{ transaction.crypto_amount|floatformat:3 }}
-		<td>{{ transaction.amount|formatcurrency }} {{ transaction.fiat_currency }}</td>
+			{% if transaction.crypto_fee and transaction.crypto_fee > 0 %}Fee: {{ transaction.crypto_fee|floatformat:3 }}{% endif %}
-		<td>{% if transaction.amount < 0 %}
+			{{ transaction.crypto_currency }}
+		</td>
+		<td>{{ transaction.fiat_amount|formatcurrency }}
+			{% if transaction.fiat_fee and transaction.fiat_fee > 0 %}Fee: {{ transaction.fiat_fee|floatformat:2 }}{% endif %}
+			{{ transaction.fiat_currency }}
+		</td>
+		<td>{% if transaction.transaction_type == 'S' %}
 			{{ transaction.exchange_rate|formatcurrency }} {{ transaction.crypto_currency }}/{{ transaction.fiat_currency }}
 			{% endif %}
 		</td>
-		<td>{% if transaction.amount > 0 %}
+		<td>{% if transaction.transaction_type == 'B' %}
 			{{ transaction.exchange_rate|formatcurrency }} {{ transaction.crypto_currency }}/{{ transaction.fiat_currency }}
 			{% endif %}
 		</td>