From 8b16281822041496e2a732281384361a127cfd89 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Sat, 13 May 2023 22:32:21 +0330 Subject: [PATCH 01/12] initialize project and its apps --- .gitignore | 160 +++++++++++++++++++++++++++++++++ Config/__init__.py | 0 Config/asgi.py | 16 ++++ Config/settings.py | 123 +++++++++++++++++++++++++ Config/urls.py | 22 +++++ Config/wsgi.py | 16 ++++ manage.py | 22 +++++ readers/__init__.py | 0 readers/admin.py | 3 + readers/apps.py | 6 ++ readers/migrations/__init__.py | 0 readers/models.py | 3 + readers/tests.py | 3 + readers/views.py | 3 + requirements.txt | 3 + users/__init__.py | 0 users/admin.py | 3 + users/apps.py | 6 ++ users/migrations/__init__.py | 0 users/models.py | 3 + users/tests.py | 3 + users/views.py | 3 + 22 files changed, 398 insertions(+) create mode 100644 .gitignore create mode 100644 Config/__init__.py create mode 100644 Config/asgi.py create mode 100644 Config/settings.py create mode 100644 Config/urls.py create mode 100644 Config/wsgi.py create mode 100755 manage.py create mode 100644 readers/__init__.py create mode 100644 readers/admin.py create mode 100644 readers/apps.py create mode 100644 readers/migrations/__init__.py create mode 100644 readers/models.py create mode 100644 readers/tests.py create mode 100644 readers/views.py create mode 100644 requirements.txt create mode 100644 users/__init__.py create mode 100644 users/admin.py create mode 100644 users/apps.py create mode 100644 users/migrations/__init__.py create mode 100644 users/models.py create mode 100644 users/tests.py create mode 100644 users/views.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6769e21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/Config/__init__.py b/Config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Config/asgi.py b/Config/asgi.py new file mode 100644 index 0000000..dfa0ff9 --- /dev/null +++ b/Config/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for Config project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Config.settings') + +application = get_asgi_application() diff --git a/Config/settings.py b/Config/settings.py new file mode 100644 index 0000000..0e858e5 --- /dev/null +++ b/Config/settings.py @@ -0,0 +1,123 @@ +""" +Django settings for Config project. + +Generated by 'django-admin startproject' using Django 4.2.1. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-5urf$zue#=!d5er1k1h!oyjg()shna+2hu))%a8#!a7z!qv+9)' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'Config.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'Config.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/Config/urls.py b/Config/urls.py new file mode 100644 index 0000000..29fdc30 --- /dev/null +++ b/Config/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for Config project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/Config/wsgi.py b/Config/wsgi.py new file mode 100644 index 0000000..a5e7fc9 --- /dev/null +++ b/Config/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for Config project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Config.settings') + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..3051634 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Config.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/readers/__init__.py b/readers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/readers/admin.py b/readers/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/readers/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/readers/apps.py b/readers/apps.py new file mode 100644 index 0000000..f0e93f5 --- /dev/null +++ b/readers/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ReadersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'readers' diff --git a/readers/migrations/__init__.py b/readers/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/readers/models.py b/readers/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/readers/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/readers/tests.py b/readers/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/readers/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/readers/views.py b/readers/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/readers/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..70eaad1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +django==4.2.1 +djangorestframework==3.14.0 +kucoin-python diff --git a/users/__init__.py b/users/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/users/admin.py b/users/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/users/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/users/apps.py b/users/apps.py new file mode 100644 index 0000000..72b1401 --- /dev/null +++ b/users/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class UsersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'users' diff --git a/users/migrations/__init__.py b/users/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/users/models.py b/users/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/users/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/users/tests.py b/users/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/users/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/users/views.py b/users/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/users/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From a66991365481aab3eb9f6954d6236c09e7b3602a Mon Sep 17 00:00:00 2001 From: alireza51 Date: Sat, 13 May 2023 22:49:49 +0330 Subject: [PATCH 02/12] simple-JWT login --- Config/settings.py | 17 ++++++++++++ Config/urls.py | 3 +- requirements.txt | 2 ++ users/api/serializers.py | 0 users/api/urls.py | 11 ++++++++ users/api/views.py | 0 users/migrations/0001_initial.py | 47 ++++++++++++++++++++++++++++++++ users/models.py | 10 ++++++- 8 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 users/api/serializers.py create mode 100644 users/api/urls.py create mode 100644 users/api/views.py create mode 100644 users/migrations/0001_initial.py diff --git a/Config/settings.py b/Config/settings.py index 0e858e5..6f63508 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -37,6 +37,13 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + + # Third-party apps + 'rest_framework', + + # Apps + 'users.apps.UsersConfig', + 'readers.apps.ReadersConfig', ] MIDDLEWARE = [ @@ -121,3 +128,13 @@ # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# Custom user model +AUTH_USER_MODEL = "users.User" + +# Rest framework +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework_simplejwt.authentication.JWTAuthentication', + ) +} diff --git a/Config/urls.py b/Config/urls.py index 29fdc30..b5eaebb 100644 --- a/Config/urls.py +++ b/Config/urls.py @@ -15,8 +15,9 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import path, include urlpatterns = [ path('admin/', admin.site.urls), + path('api/', include('users.api.urls')) ] diff --git a/requirements.txt b/requirements.txt index 70eaad1..699c3e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ django==4.2.1 djangorestframework==3.14.0 kucoin-python +djangorestframework-simplejwt +django-filter diff --git a/users/api/serializers.py b/users/api/serializers.py new file mode 100644 index 0000000..e69de29 diff --git a/users/api/urls.py b/users/api/urls.py new file mode 100644 index 0000000..f46344c --- /dev/null +++ b/users/api/urls.py @@ -0,0 +1,11 @@ +from django.urls import path, include +from rest_framework_simplejwt.views import ( + TokenObtainPairView, + TokenRefreshView, +) + + +urlpatterns = [ + path('login/token/', TokenObtainPairView.as_view(), name='token_obtain'), + path('login/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), +] diff --git a/users/api/views.py b/users/api/views.py new file mode 100644 index 0000000..e69de29 diff --git a/users/migrations/0001_initial.py b/users/migrations/0001_initial.py new file mode 100644 index 0000000..d4b98f1 --- /dev/null +++ b/users/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.1 on 2023-05-13 19:12 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('api_key', models.CharField(max_length=255)), + ('api_secret', models.CharField(max_length=255)), + ('api_passphrase', models.CharField(max_length=255)), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/users/models.py b/users/models.py index 71a8362..4947a27 100644 --- a/users/models.py +++ b/users/models.py @@ -1,3 +1,11 @@ from django.db import models -# Create your models here. +from django.contrib.auth.models import AbstractUser + +class User(AbstractUser): + api_key = models.CharField(max_length=255, null=False, blank=False) + api_secret = models.CharField(max_length=255, null=False, blank=False) + api_passphrase = models.CharField(max_length=255, null=False, blank=False) + + def __str__(self) -> str: + return f'{self.id}. {self.first_name} {self.last_name}' From e3549193237a3751bc0d9e102527f8875e1e8c4f Mon Sep 17 00:00:00 2001 From: alireza51 Date: Sun, 14 May 2023 12:43:40 +0330 Subject: [PATCH 03/12] user api --- Config/urls.py | 2 +- users/admin.py | 4 +++- users/api/routers.py | 6 ++++++ users/api/serializers.py | 27 +++++++++++++++++++++++++++ users/api/urls.py | 7 +++++-- users/api/views.py | 23 +++++++++++++++++++++++ 6 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 users/api/routers.py diff --git a/Config/urls.py b/Config/urls.py index b5eaebb..c0cc667 100644 --- a/Config/urls.py +++ b/Config/urls.py @@ -19,5 +19,5 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('api/', include('users.api.urls')) + path('api/account/', include('users.api.urls')) ] diff --git a/users/admin.py b/users/admin.py index 8c38f3f..bca5cc5 100644 --- a/users/admin.py +++ b/users/admin.py @@ -1,3 +1,5 @@ from django.contrib import admin +from django.contrib.auth import get_user_model -# Register your models here. +User = get_user_model() +admin.site.register(User) diff --git a/users/api/routers.py b/users/api/routers.py new file mode 100644 index 0000000..e3188f3 --- /dev/null +++ b/users/api/routers.py @@ -0,0 +1,6 @@ +from rest_framework import routers +from .views import UserCreateView, UserGetView + +router = routers.SimpleRouter() +router.register('signup', UserCreateView, basename='api-user-signup') +router.register('', UserGetView, basename='api-account') \ No newline at end of file diff --git a/users/api/serializers.py b/users/api/serializers.py index e69de29..e20e056 100644 --- a/users/api/serializers.py +++ b/users/api/serializers.py @@ -0,0 +1,27 @@ +from rest_framework import serializers +from django.contrib.auth.hashers import make_password +from django.contrib.auth import get_user_model + +User = get_user_model() + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = [ + 'id', + 'username', + 'first_name', + 'last_name', + 'api_key', + 'api_secret', + 'api_passphrase', + 'password', + ] + extra_kwargs = { + 'password': {'write_only': True}, + 'id': {'read_only': True}, + } + + def create(self, validated_data): + validated_data['password'] = make_password(validated_data['password']) + return User.objects.create(**validated_data) diff --git a/users/api/urls.py b/users/api/urls.py index f46344c..e13e1b1 100644 --- a/users/api/urls.py +++ b/users/api/urls.py @@ -3,9 +3,12 @@ TokenObtainPairView, TokenRefreshView, ) +from .routers import router urlpatterns = [ - path('login/token/', TokenObtainPairView.as_view(), name='token_obtain'), - path('login/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), + path('login/token/', TokenObtainPairView.as_view(), name='api-token_obtain'), + path('login/token/refresh/', TokenRefreshView.as_view(), name='api-token_refresh'), ] + +urlpatterns += router.urls \ No newline at end of file diff --git a/users/api/views.py b/users/api/views.py index e69de29..b395553 100644 --- a/users/api/views.py +++ b/users/api/views.py @@ -0,0 +1,23 @@ +from rest_framework import generics, mixins, views, viewsets, permissions +from django.contrib.auth import get_user_model + +from .serializers import UserSerializer + +User = get_user_model() + +class UserCreateView(viewsets.GenericViewSet, mixins.CreateModelMixin): + serializer_class = UserSerializer + authentication_classes = [] + permission_classes = [] + queryset = User.objects.all() + +class UserGetView(viewsets.GenericViewSet, mixins.RetrieveModelMixin): + serializer_class = UserSerializer + authentication_classes = [] + # permission_classes = [permissions.IsAuthenticated,] + + def get_queryset(self): + if self.request.user.is_staff: + return User.objects.all() + else: + return User.objects.get(pk=self.request.user.id) \ No newline at end of file From b4f262a02e3b3b5272655942b22df3332d4b9c11 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Sun, 14 May 2023 15:01:44 +0330 Subject: [PATCH 04/12] user authorization --- Config/settings.py | 43 +++++++++++++++++++++++++++++++++++++++++++ users/api/views.py | 7 ++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Config/settings.py b/Config/settings.py index 6f63508..a230835 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -11,6 +11,7 @@ """ from pathlib import Path +from datetime import timedelta # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -138,3 +139,45 @@ 'rest_framework_simplejwt.authentication.JWTAuthentication', ) } + +# Simple-JWT +SIMPLE_JWT = { + "ACCESS_TOKEN_LIFETIME": timedelta(days=30), + "REFRESH_TOKEN_LIFETIME": timedelta(days=30), + # "ROTATE_REFRESH_TOKENS": False, + # "BLACKLIST_AFTER_ROTATION": False, + # "UPDATE_LAST_LOGIN": False, + + "ALGORITHM": "HS256", + "SIGNING_KEY": SECRET_KEY, + # "VERIFYING_KEY": "", + # "AUDIENCE": None, + # "ISSUER": None, + # "JSON_ENCODER": None, + # "JWK_URL": None, + # "LEEWAY": 0, + + "AUTH_HEADER_TYPES": ("JWT",), + "AUTH_HEADER_NAME": "HTTP_AUTHORIZATION", + # "USER_ID_FIELD": "id", + # "USER_ID_CLAIM": "user_id", + # "USER_AUTHENTICATION_RULE": "rest_framework_simplejwt.authentication.default_user_authentication_rule", + + # "AUTH_TOKEN_CLASSES": ("rest_framework_simplejwt.tokens.AccessToken",), + # "TOKEN_TYPE_CLAIM": "token_type", + # "TOKEN_USER_CLASS": "rest_framework_simplejwt.models.TokenUser", + + # "JTI_CLAIM": "jti", + + # "SLIDING_TOKEN_REFRESH_EXP_CLAIM": "refresh_exp", + # "SLIDING_TOKEN_LIFETIME": timedelta(minutes=5), + # "SLIDING_TOKEN_REFRESH_LIFETIME": timedelta(days=1), + + # "TOKEN_OBTAIN_SERIALIZER": "rest_framework_simplejwt.serializers.TokenObtainPairSerializer", + # "TOKEN_REFRESH_SERIALIZER": "rest_framework_simplejwt.serializers.TokenRefreshSerializer", + # "TOKEN_VERIFY_SERIALIZER": "rest_framework_simplejwt.serializers.TokenVerifySerializer", + # "TOKEN_BLACKLIST_SERIALIZER": "rest_framework_simplejwt.serializers.TokenBlacklistSerializer", + # "SLIDING_TOKEN_OBTAIN_SERIALIZER": "rest_framework_simplejwt.serializers.TokenObtainSlidingSerializer", + # "SLIDING_TOKEN_REFRESH_SERIALIZER": "rest_framework_simplejwt.serializers.TokenRefreshSlidingSerializer", +} + diff --git a/users/api/views.py b/users/api/views.py index b395553..71a6a43 100644 --- a/users/api/views.py +++ b/users/api/views.py @@ -13,11 +13,12 @@ class UserCreateView(viewsets.GenericViewSet, mixins.CreateModelMixin): class UserGetView(viewsets.GenericViewSet, mixins.RetrieveModelMixin): serializer_class = UserSerializer - authentication_classes = [] - # permission_classes = [permissions.IsAuthenticated,] + permission_classes = [permissions.IsAuthenticated,] def get_queryset(self): + print(self.request.user.id) + print() if self.request.user.is_staff: return User.objects.all() else: - return User.objects.get(pk=self.request.user.id) \ No newline at end of file + return User.objects.filter(pk=self.request.user.id) \ No newline at end of file From 29d22df71ac4550a1cb958fedef54a4ba864f36c Mon Sep 17 00:00:00 2001 From: alireza51 Date: Sun, 14 May 2023 16:24:50 +0330 Subject: [PATCH 05/12] encryption of api details --- .gitignore | 1 + Config/settings.py | 3 ++ requirements.txt | 1 + users/api/serializers.py | 18 ++++++++ ..._key_alter_user_api_passphrase_and_more.py | 28 +++++++++++++ users/models.py | 6 +-- utils/__init__.py | 0 utils/encryption.py | 42 +++++++++++++++++++ 8 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 users/migrations/0002_alter_user_api_key_alter_user_api_passphrase_and_more.py create mode 100644 utils/__init__.py create mode 100644 utils/encryption.py diff --git a/.gitignore b/.gitignore index 6769e21..2dcf51a 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ venv/ ENV/ env.bak/ venv.bak/ +env # Spyder project settings .spyderproject diff --git a/Config/settings.py b/Config/settings.py index a230835..fe61544 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -23,6 +23,9 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'django-insecure-5urf$zue#=!d5er1k1h!oyjg()shna+2hu))%a8#!a7z!qv+9)' +# Fernet key used for encrypting user secrects +ENCRYPTION_KEY = b'8TpVNaD5OCy01Fz3gpZ3k9m2jhl51kMhHCn7ARCEiF4=' + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/requirements.txt b/requirements.txt index 699c3e8..5c2a9ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ djangorestframework==3.14.0 kucoin-python djangorestframework-simplejwt django-filter +cryptography diff --git a/users/api/serializers.py b/users/api/serializers.py index e20e056..24d66b2 100644 --- a/users/api/serializers.py +++ b/users/api/serializers.py @@ -2,6 +2,8 @@ from django.contrib.auth.hashers import make_password from django.contrib.auth import get_user_model +from utils.encryption import encrypt, decrypt + User = get_user_model() class UserSerializer(serializers.ModelSerializer): @@ -23,5 +25,21 @@ class Meta: } def create(self, validated_data): + """ + Hash password and encrypt api detail for safety. + """ validated_data['password'] = make_password(validated_data['password']) + validated_data['api_key'] = encrypt(validated_data['api_key']) + validated_data['api_secret'] = encrypt(validated_data['api_secret']) + validated_data['api_passphrase'] = encrypt(validated_data['api_passphrase']) return User.objects.create(**validated_data) + + def to_representation(self, instance): + """ + Decrypt api details to show to user. + """ + ret = super().to_representation(instance) + ret['api_key'] = decrypt(ret['api_key']) + ret['api_secret'] = decrypt(ret['api_secret']) + ret['api_passphrase'] = decrypt(ret['api_passphrase']) + return ret diff --git a/users/migrations/0002_alter_user_api_key_alter_user_api_passphrase_and_more.py b/users/migrations/0002_alter_user_api_key_alter_user_api_passphrase_and_more.py new file mode 100644 index 0000000..d5e598d --- /dev/null +++ b/users/migrations/0002_alter_user_api_key_alter_user_api_passphrase_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.1 on 2023-05-14 12:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='api_key', + field=models.CharField(max_length=512), + ), + migrations.AlterField( + model_name='user', + name='api_passphrase', + field=models.CharField(max_length=512), + ), + migrations.AlterField( + model_name='user', + name='api_secret', + field=models.CharField(max_length=512), + ), + ] diff --git a/users/models.py b/users/models.py index 4947a27..a69a476 100644 --- a/users/models.py +++ b/users/models.py @@ -3,9 +3,9 @@ from django.contrib.auth.models import AbstractUser class User(AbstractUser): - api_key = models.CharField(max_length=255, null=False, blank=False) - api_secret = models.CharField(max_length=255, null=False, blank=False) - api_passphrase = models.CharField(max_length=255, null=False, blank=False) + api_key = models.CharField(max_length=512, null=False, blank=False) + api_secret = models.CharField(max_length=512, null=False, blank=False) + api_passphrase = models.CharField(max_length=512, null=False, blank=False) def __str__(self) -> str: return f'{self.id}. {self.first_name} {self.last_name}' diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/encryption.py b/utils/encryption.py new file mode 100644 index 0000000..06768c8 --- /dev/null +++ b/utils/encryption.py @@ -0,0 +1,42 @@ +from django.conf import settings +from cryptography.fernet import Fernet +import base64 +import logging +import traceback + +# get the key from settings +f = Fernet(settings.ENCRYPTION_KEY) + +def encrypt(raw_txt): + """ + Function to encrypt raw text using Fernet class + """ + try: + # convert integer etc to string first + txt = str(raw_txt) + #input should be byte, so convert the text to byte + encrypted_text = f.encrypt(txt.encode('ascii')) + # encode to urlsafe base64 format + encrypted_text = base64.urlsafe_b64encode(encrypted_text).decode("ascii") + return encrypted_text + except Exception as e: + # log the error if any + logging.getLogger("error_logger").error(traceback.format_exc()) + return None + + +def decrypt(encrypted_txt): + """ + Function to decrypt encrypted text using Fernet class + """ + try: + # base64 decode + txt = base64.urlsafe_b64decode(encrypted_txt) + # decryption + decoded_text = f.decrypt(txt).decode("ascii") + return decoded_text + except Exception as e: + # log the error + logging.getLogger("error_logger").error(traceback.format_exc()) + return None + From e723f20750b320259cab0ae0e5079c0559e9262b Mon Sep 17 00:00:00 2001 From: alireza51 Date: Mon, 15 May 2023 17:27:36 +0330 Subject: [PATCH 06/12] redis celery priodic task --- .gitignore | 5 ++- Config/__init__.py | 3 ++ Config/celery.py | 22 +++++++++++ Config/settings.py | 17 +++++++++ Config/urls.py | 3 +- readers/api/router.py | 5 +++ readers/api/serializers.py | 14 +++++++ readers/api/urls.py | 7 ++++ readers/api/views.py | 20 ++++++++++ readers/migrations/0001_initial.py | 30 +++++++++++++++ readers/models.py | 15 +++++++- readers/tasks.py | 17 +++++++++ requirements.txt | 1 + users/api/serializers.py | 2 +- users/api/views.py | 4 +- utils/encryption.py | 9 ++++- utils/request.py | 59 ++++++++++++++++++++++++++++++ 17 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 Config/celery.py create mode 100644 readers/api/router.py create mode 100644 readers/api/serializers.py create mode 100644 readers/api/urls.py create mode 100644 readers/api/views.py create mode 100644 readers/migrations/0001_initial.py create mode 100644 readers/tasks.py create mode 100644 utils/request.py diff --git a/.gitignore b/.gitignore index 2dcf51a..4d2d22f 100644 --- a/.gitignore +++ b/.gitignore @@ -158,4 +158,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +dropped.py +req.txt diff --git a/Config/__init__.py b/Config/__init__.py index e69de29..cd04264 100644 --- a/Config/__init__.py +++ b/Config/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/Config/celery.py b/Config/celery.py new file mode 100644 index 0000000..4d6f81e --- /dev/null +++ b/Config/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Config.settings') + +app = Celery('Config') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') \ No newline at end of file diff --git a/Config/settings.py b/Config/settings.py index fe61544..c52bd14 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -184,3 +184,20 @@ # "SLIDING_TOKEN_REFRESH_SERIALIZER": "rest_framework_simplejwt.serializers.TokenRefreshSlidingSerializer", } +# Celery configs +CELERY_BROKER_URL = 'redis://localhost:6379' +CELERY_RESULT_BACKEND = 'redis://localhost:6379' +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_ENABLE_UTC = True +CELERY_IMPORTS = [ + 'readers.tasks', +] + +# Celery settings +CELERY_BEAT_SCHEDULE = { + 'every-30-sec': { + 'task': 'readers.tasks.cach_positions_all_users', + 'schedule': 30.0 + } +} \ No newline at end of file diff --git a/Config/urls.py b/Config/urls.py index c0cc667..99fc34b 100644 --- a/Config/urls.py +++ b/Config/urls.py @@ -19,5 +19,6 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('api/account/', include('users.api.urls')) + path('api/user/', include('users.api.urls')), + path('api/', include('readers.api.urls')) ] diff --git a/readers/api/router.py b/readers/api/router.py new file mode 100644 index 0000000..a0c948a --- /dev/null +++ b/readers/api/router.py @@ -0,0 +1,5 @@ +from rest_framework import routers +from .views import AccountViewSet + +router = routers.SimpleRouter() +router.register('positions', AccountViewSet, basename='api-positions') \ No newline at end of file diff --git a/readers/api/serializers.py b/readers/api/serializers.py new file mode 100644 index 0000000..773af0a --- /dev/null +++ b/readers/api/serializers.py @@ -0,0 +1,14 @@ +from rest_framework import serializers +from readers.models import Account + +class AccountSerializer(serializers.ModelSerializer): + class Meta: + model = Account + fields = [ + 'account_id', + 'currency', + 'account_type', + 'balance', + 'available', + 'holds' + ] \ No newline at end of file diff --git a/readers/api/urls.py b/readers/api/urls.py new file mode 100644 index 0000000..dba5c60 --- /dev/null +++ b/readers/api/urls.py @@ -0,0 +1,7 @@ +from django.urls import path, include +from .router import router + +urlpatterns = [ + +] +urlpatterns += router.urls \ No newline at end of file diff --git a/readers/api/views.py b/readers/api/views.py new file mode 100644 index 0000000..d7a5e6b --- /dev/null +++ b/readers/api/views.py @@ -0,0 +1,20 @@ +from rest_framework import viewsets, permissions, mixins +from readers.models import Account +from .serializers import AccountSerializer + +from utils.request import update_positions_for_user + +class AccountViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): + serializer_class = AccountSerializer + permission_classes = [permissions.IsAuthenticated,] + + def get_queryset(self): + account = self.request.user.accounts.all() + if account: + return account + else: + return Account.objects.none() + + def list(self, request, *args, **kwargs): + update_positions_for_user(request.user) + return super().list(request, *args, **kwargs) \ No newline at end of file diff --git a/readers/migrations/0001_initial.py b/readers/migrations/0001_initial.py new file mode 100644 index 0000000..b2c0d63 --- /dev/null +++ b/readers/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.1 on 2023-05-15 09:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Account', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('account_id', models.CharField(max_length=512)), + ('currency', models.CharField(max_length=10)), + ('account_type', models.CharField(max_length=6)), + ('balance', models.DecimalField(decimal_places=6, max_digits=20)), + ('available', models.DecimalField(decimal_places=6, max_digits=20)), + ('holds', models.DecimalField(decimal_places=6, max_digits=20)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='accounts', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/readers/models.py b/readers/models.py index 71a8362..77cb6e0 100644 --- a/readers/models.py +++ b/readers/models.py @@ -1,3 +1,16 @@ from django.db import models +from django.contrib.auth import get_user_model -# Create your models here. +User = get_user_model() + +class Account(models.Model): + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='accounts') + account_id = models.CharField(max_length=512, null=False, blank=False) + currency = models.CharField(max_length=10, null=False, blank=False) + account_type = models.CharField(max_length=6, null=False, blank=False) + balance = models.DecimalField(max_digits=20, decimal_places=6) + available = models.DecimalField(max_digits=20, decimal_places=6) + holds = models.DecimalField(max_digits=20, decimal_places=6) + + def __str__(self) -> str: + return f'{self.account_id} - user:{self.user.id}' \ No newline at end of file diff --git a/readers/tasks.py b/readers/tasks.py new file mode 100644 index 0000000..e19aa0f --- /dev/null +++ b/readers/tasks.py @@ -0,0 +1,17 @@ +from celery.utils.log import get_task_logger +from celery import shared_task +from django.contrib.auth import get_user_model +from multiprocessing.dummy import Pool as ThreadPool + +from utils.request import cache_position_in_redis + +logger = get_task_logger(__name__) +User = get_user_model() + +@shared_task +def cach_positions_all_users(): + users = User.objects.all() + pool = ThreadPool(4) + params = list(users) + results = pool.map(cache_position_in_redis, params) + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 5c2a9ef..7ddf8c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ kucoin-python djangorestframework-simplejwt django-filter cryptography +celery==4.4.2 diff --git a/users/api/serializers.py b/users/api/serializers.py index 24d66b2..baefd42 100644 --- a/users/api/serializers.py +++ b/users/api/serializers.py @@ -26,7 +26,7 @@ class Meta: def create(self, validated_data): """ - Hash password and encrypt api detail for safety. + Hash password and encrypt api details for safety. """ validated_data['password'] = make_password(validated_data['password']) validated_data['api_key'] = encrypt(validated_data['api_key']) diff --git a/users/api/views.py b/users/api/views.py index 71a6a43..4ccae74 100644 --- a/users/api/views.py +++ b/users/api/views.py @@ -1,4 +1,4 @@ -from rest_framework import generics, mixins, views, viewsets, permissions +from rest_framework import mixins, viewsets, permissions from django.contrib.auth import get_user_model from .serializers import UserSerializer @@ -16,8 +16,6 @@ class UserGetView(viewsets.GenericViewSet, mixins.RetrieveModelMixin): permission_classes = [permissions.IsAuthenticated,] def get_queryset(self): - print(self.request.user.id) - print() if self.request.user.is_staff: return User.objects.all() else: diff --git a/utils/encryption.py b/utils/encryption.py index 06768c8..7eda583 100644 --- a/utils/encryption.py +++ b/utils/encryption.py @@ -1,3 +1,4 @@ +from typing import Optional from django.conf import settings from cryptography.fernet import Fernet import base64 @@ -7,9 +8,11 @@ # get the key from settings f = Fernet(settings.ENCRYPTION_KEY) -def encrypt(raw_txt): +def encrypt(raw_txt:str) -> Optional[str]: """ Function to encrypt raw text using Fernet class + params: + - raw_txt : raw string to be ecrypted """ try: # convert integer etc to string first @@ -25,9 +28,11 @@ def encrypt(raw_txt): return None -def decrypt(encrypted_txt): +def decrypt(encrypted_txt:str) -> Optional[str]: """ Function to decrypt encrypted text using Fernet class + params: + - encrypted_txt : ecrypted text to be decrypted """ try: # base64 decode diff --git a/utils/request.py b/utils/request.py new file mode 100644 index 0000000..caf5ff5 --- /dev/null +++ b/utils/request.py @@ -0,0 +1,59 @@ +from kucoin.client import User as KuUser +from django.contrib.auth import get_user_model +from readers.models import Account +import redis +import pickle + +from .encryption import decrypt + +User = get_user_model() + +def get_account_for_user(user:User) -> list: + client = KuUser(decrypt(user.api_key), decrypt(user.api_secret), decrypt(user.api_passphrase)) + # Request for accounts using Kucoin SDK + account_list = client.get_account_list() + # Transaction success + if account_list['code'] == '200000': + return account_list['data'] + else: + raise Exception('Error connecting to server') + +def update_positions_for_user(user:User) -> None: + """ + Update all positions in account of user + params: + - user : user + """ + account_list = get_account_for_user(user) + bulk_accounts = [] + # make a bulk list to for create_bulk + for item in account_list: + account = Account(**item) + account.user = user + bulk_accounts.append(account) + # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature + Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) + +def load_positions_from_cach(user:User) -> None: + """ + Reads all positions in account of user + params: + - user : user + """ + # account_list = get_account_for_user(user) + # bulk_accounts = [] + # # make a bulk list to for create_bulk + # for item in account_list: + # account = Account(**item) + # account.user = user + # bulk_accounts.append(account) + # # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature + # Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) + +def cache_position_in_redis(user:User) -> None: + """ + Handles caching positions in redis + """ + account_list = get_account_for_user(user) + client = redis.Redis('localhost', port=6379, db=0) + client.set(f'user_{user.id}', pickle.dumps(account_list)) From 4df1f9d35681430b2ab8ff1188f3dbd33c5240ce Mon Sep 17 00:00:00 2001 From: alireza51 Date: Mon, 15 May 2023 19:40:38 +0330 Subject: [PATCH 07/12] cache account in redis and sync every 30s --- readers/api/views.py | 22 +++++++++--------- readers/models.py | 3 +++ readers/tasks.py | 4 ++-- utils/cache.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ utils/request.py | 45 +----------------------------------- 5 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 utils/cache.py diff --git a/readers/api/views.py b/readers/api/views.py index d7a5e6b..608e331 100644 --- a/readers/api/views.py +++ b/readers/api/views.py @@ -1,20 +1,20 @@ from rest_framework import viewsets, permissions, mixins +from rest_framework.response import Response from readers.models import Account from .serializers import AccountSerializer - -from utils.request import update_positions_for_user +from utils.cache import read_positions_from_cache class AccountViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): serializer_class = AccountSerializer permission_classes = [permissions.IsAuthenticated,] - - def get_queryset(self): - account = self.request.user.accounts.all() - if account: - return account - else: - return Account.objects.none() def list(self, request, *args, **kwargs): - update_positions_for_user(request.user) - return super().list(request, *args, **kwargs) \ No newline at end of file + accounts = read_positions_from_cache(user=request.user) + if accounts == 'DB': + qs = request.user.accounts.all() + serializer = self.get_serializer(qs, many=True) + return Response(serializer.data) + if accounts: + serializer = self.get_serializer(accounts) + return Response(serializer.data) + return Response({}) diff --git a/readers/models.py b/readers/models.py index 77cb6e0..8c7060f 100644 --- a/readers/models.py +++ b/readers/models.py @@ -12,5 +12,8 @@ class Account(models.Model): available = models.DecimalField(max_digits=20, decimal_places=6) holds = models.DecimalField(max_digits=20, decimal_places=6) + class Meta: + unique_together = ('user', 'account_id',) + def __str__(self) -> str: return f'{self.account_id} - user:{self.user.id}' \ No newline at end of file diff --git a/readers/tasks.py b/readers/tasks.py index e19aa0f..2994e98 100644 --- a/readers/tasks.py +++ b/readers/tasks.py @@ -3,7 +3,7 @@ from django.contrib.auth import get_user_model from multiprocessing.dummy import Pool as ThreadPool -from utils.request import cache_position_in_redis +from utils.cache import write_positions_in_cache logger = get_task_logger(__name__) User = get_user_model() @@ -13,5 +13,5 @@ def cach_positions_all_users(): users = User.objects.all() pool = ThreadPool(4) params = list(users) - results = pool.map(cache_position_in_redis, params) + results = pool.map(write_positions_in_cache, params) \ No newline at end of file diff --git a/utils/cache.py b/utils/cache.py new file mode 100644 index 0000000..fada761 --- /dev/null +++ b/utils/cache.py @@ -0,0 +1,55 @@ +from readers.models import Account +from django.contrib.auth import get_user_model + +import redis +import pickle + +from utils.request import get_account_for_user + +User = get_user_model() + +def write_positions_in_cache(user:User) -> None: + """ + Handles caching positions in redis + If redis was down it will call another function to store data in DB. + params: + user : user + """ + account_list = get_account_for_user(user) + try: + client = redis.Redis('localhost', port=6379, db=0) + client.set(f'user_{user.id}', pickle.dumps(account_list)) + except redis.exceptions.ConnectionError: # Rides down store in DB + write_to_db(user, account_list) + +def read_positions_from_cache(user:User) -> list: + """ + Reads all positions in account of user + params: + - user : user + """ + try: + client = redis.Redis('localhost', port=6379, db=0) + account_list = client.get(f'user_{user.id}') # returns pickled + if account_list: + return pickle.loads(account_list) + else: + return list() + except redis.exceptions.ConnectionError: # Rides down store in DB + return 'DB' # View handles reading from DB if necessary + + +def write_to_db(user:User, account_list:list) -> None: + """ + Function to replace storing positions in DB + params: + user : user + account_list : list of position (returned by Kucoin) + """ + bulk_accounts = [] + for item in account_list: + account = Account(**item) + account.user = user + bulk_accounts.append(account) + # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature + Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) diff --git a/utils/request.py b/utils/request.py index caf5ff5..29a2b0d 100644 --- a/utils/request.py +++ b/utils/request.py @@ -1,8 +1,5 @@ from kucoin.client import User as KuUser from django.contrib.auth import get_user_model -from readers.models import Account -import redis -import pickle from .encryption import decrypt @@ -16,44 +13,4 @@ def get_account_for_user(user:User) -> list: if account_list['code'] == '200000': return account_list['data'] else: - raise Exception('Error connecting to server') - -def update_positions_for_user(user:User) -> None: - """ - Update all positions in account of user - params: - - user : user - """ - account_list = get_account_for_user(user) - bulk_accounts = [] - # make a bulk list to for create_bulk - for item in account_list: - account = Account(**item) - account.user = user - bulk_accounts.append(account) - # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature - Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) - -def load_positions_from_cach(user:User) -> None: - """ - Reads all positions in account of user - params: - - user : user - """ - # account_list = get_account_for_user(user) - # bulk_accounts = [] - # # make a bulk list to for create_bulk - # for item in account_list: - # account = Account(**item) - # account.user = user - # bulk_accounts.append(account) - # # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature - # Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) - -def cache_position_in_redis(user:User) -> None: - """ - Handles caching positions in redis - """ - account_list = get_account_for_user(user) - client = redis.Redis('localhost', port=6379, db=0) - client.set(f'user_{user.id}', pickle.dumps(account_list)) + raise Exception('Error connecting to Kucoin server') From ef757e95fd1354ad1843d3ae96cd67b5266eedc9 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Mon, 15 May 2023 19:50:23 +0330 Subject: [PATCH 08/12] get number of thread in celery worker from settings enviornment --- Config/settings.py | 5 ++++- readers/tasks.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Config/settings.py b/Config/settings.py index c52bd14..5a42ccd 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -200,4 +200,7 @@ 'task': 'readers.tasks.cach_positions_all_users', 'schedule': 30.0 } -} \ No newline at end of file +} + +# Thread pool number for celery worker threads +THREAD_POOL = 4 \ No newline at end of file diff --git a/readers/tasks.py b/readers/tasks.py index 2994e98..dbed70e 100644 --- a/readers/tasks.py +++ b/readers/tasks.py @@ -2,6 +2,7 @@ from celery import shared_task from django.contrib.auth import get_user_model from multiprocessing.dummy import Pool as ThreadPool +from django.conf import settings from utils.cache import write_positions_in_cache @@ -11,7 +12,7 @@ @shared_task def cach_positions_all_users(): users = User.objects.all() - pool = ThreadPool(4) + pool = ThreadPool(settings.THREAD_POOL) params = list(users) results = pool.map(write_positions_in_cache, params) \ No newline at end of file From adb38fc226138d410b4ca1f789f19c300ab468f8 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Mon, 15 May 2023 19:55:49 +0330 Subject: [PATCH 09/12] edit doc string of functions --- utils/request.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/request.py b/utils/request.py index 29a2b0d..68a15b4 100644 --- a/utils/request.py +++ b/utils/request.py @@ -6,11 +6,13 @@ User = get_user_model() def get_account_for_user(user:User) -> list: + """ + Function to handle Kucoin's account requests + params: + - user : user + """ client = KuUser(decrypt(user.api_key), decrypt(user.api_secret), decrypt(user.api_passphrase)) # Request for accounts using Kucoin SDK account_list = client.get_account_list() # Transaction success - if account_list['code'] == '200000': - return account_list['data'] - else: - raise Exception('Error connecting to Kucoin server') + return account_list['data'] From e0977c718e92f3e3b02ecf152499abc158fbe175 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Tue, 16 May 2023 14:01:47 +0330 Subject: [PATCH 10/12] debug models and serializers after testing with real world data --- Config/settings.py | 2 +- readers/api/serializers.py | 4 ++-- readers/api/views.py | 9 +++++---- readers/migrations/0001_initial.py | 11 +++++++---- .../migrations/0002_alter_account_id_field.py | 18 ++++++++++++++++++ readers/models.py | 10 ++++++---- readers/tasks.py | 2 +- utils/request.py | 4 +++- 8 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 readers/migrations/0002_alter_account_id_field.py diff --git a/Config/settings.py b/Config/settings.py index 5a42ccd..d0633e4 100644 --- a/Config/settings.py +++ b/Config/settings.py @@ -197,7 +197,7 @@ # Celery settings CELERY_BEAT_SCHEDULE = { 'every-30-sec': { - 'task': 'readers.tasks.cach_positions_all_users', + 'task': 'readers.tasks.cache_positions_all_users', 'schedule': 30.0 } } diff --git a/readers/api/serializers.py b/readers/api/serializers.py index 773af0a..b564cab 100644 --- a/readers/api/serializers.py +++ b/readers/api/serializers.py @@ -5,9 +5,9 @@ class AccountSerializer(serializers.ModelSerializer): class Meta: model = Account fields = [ - 'account_id', + 'id', 'currency', - 'account_type', + 'type', 'balance', 'available', 'holds' diff --git a/readers/api/views.py b/readers/api/views.py index 608e331..03d902d 100644 --- a/readers/api/views.py +++ b/readers/api/views.py @@ -14,7 +14,8 @@ def list(self, request, *args, **kwargs): qs = request.user.accounts.all() serializer = self.get_serializer(qs, many=True) return Response(serializer.data) - if accounts: - serializer = self.get_serializer(accounts) - return Response(serializer.data) - return Response({}) + qs = [] + for account in accounts: + qs.append(Account(**account)) + serializer = self.get_serializer(qs, many=True) + return Response(serializer.data) diff --git a/readers/migrations/0001_initial.py b/readers/migrations/0001_initial.py index b2c0d63..168531d 100644 --- a/readers/migrations/0001_initial.py +++ b/readers/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.1 on 2023-05-15 09:16 +# Generated by Django 4.2.1 on 2023-05-16 10:16 from django.conf import settings from django.db import migrations, models @@ -17,14 +17,17 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Account', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('account_id', models.CharField(max_length=512)), + ('id', models.CharField(max_length=512)), ('currency', models.CharField(max_length=10)), - ('account_type', models.CharField(max_length=6)), + ('type', models.CharField(max_length=6)), ('balance', models.DecimalField(decimal_places=6, max_digits=20)), ('available', models.DecimalField(decimal_places=6, max_digits=20)), ('holds', models.DecimalField(decimal_places=6, max_digits=20)), + ('id_field', models.AutoField(default=79, primary_key=True, serialize=False)), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='accounts', to=settings.AUTH_USER_MODEL)), ], + options={ + 'unique_together': {('user', 'id')}, + }, ), ] diff --git a/readers/migrations/0002_alter_account_id_field.py b/readers/migrations/0002_alter_account_id_field.py new file mode 100644 index 0000000..e6a1fde --- /dev/null +++ b/readers/migrations/0002_alter_account_id_field.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-16 10:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('readers', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='account', + name='id_field', + field=models.AutoField(primary_key=True, serialize=False), + ), + ] diff --git a/readers/models.py b/readers/models.py index 8c7060f..5cd9591 100644 --- a/readers/models.py +++ b/readers/models.py @@ -1,19 +1,21 @@ from django.db import models from django.contrib.auth import get_user_model - +import random User = get_user_model() class Account(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='accounts') - account_id = models.CharField(max_length=512, null=False, blank=False) + id = models.CharField(max_length=512, null=False, blank=False) currency = models.CharField(max_length=10, null=False, blank=False) - account_type = models.CharField(max_length=6, null=False, blank=False) + type = models.CharField(max_length=6, null=False, blank=False) balance = models.DecimalField(max_digits=20, decimal_places=6) available = models.DecimalField(max_digits=20, decimal_places=6) holds = models.DecimalField(max_digits=20, decimal_places=6) + id_field = models.AutoField(primary_key=True) + class Meta: - unique_together = ('user', 'account_id',) + unique_together = ('user', 'id',) def __str__(self) -> str: return f'{self.account_id} - user:{self.user.id}' \ No newline at end of file diff --git a/readers/tasks.py b/readers/tasks.py index dbed70e..3b07468 100644 --- a/readers/tasks.py +++ b/readers/tasks.py @@ -10,7 +10,7 @@ User = get_user_model() @shared_task -def cach_positions_all_users(): +def cache_positions_all_users(): users = User.objects.all() pool = ThreadPool(settings.THREAD_POOL) params = list(users) diff --git a/utils/request.py b/utils/request.py index 68a15b4..be5ce53 100644 --- a/utils/request.py +++ b/utils/request.py @@ -15,4 +15,6 @@ def get_account_for_user(user:User) -> list: # Request for accounts using Kucoin SDK account_list = client.get_account_list() # Transaction success - return account_list['data'] + if isinstance(account_list, list): + return account_list + return [] From c6151bd50eef759c6d89a427639ebac6e8e54d05 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Tue, 16 May 2023 18:08:08 +0330 Subject: [PATCH 11/12] edit docstrings and comments --- Config/urls.py | 4 ++-- readers/api/urls.py | 4 +--- readers/api/views.py | 5 +++++ readers/tasks.py | 4 ++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Config/urls.py b/Config/urls.py index 99fc34b..8b0dd85 100644 --- a/Config/urls.py +++ b/Config/urls.py @@ -19,6 +19,6 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('api/user/', include('users.api.urls')), - path('api/', include('readers.api.urls')) + path('api/user/', include('users.api.urls')), # User actions + path('api/', include('readers.api.urls')), # Reader actions ] diff --git a/readers/api/urls.py b/readers/api/urls.py index dba5c60..e132236 100644 --- a/readers/api/urls.py +++ b/readers/api/urls.py @@ -1,7 +1,5 @@ from django.urls import path, include from .router import router -urlpatterns = [ - -] +urlpatterns = [] urlpatterns += router.urls \ No newline at end of file diff --git a/readers/api/views.py b/readers/api/views.py index 03d902d..aa76330 100644 --- a/readers/api/views.py +++ b/readers/api/views.py @@ -9,6 +9,11 @@ class AccountViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): permission_classes = [permissions.IsAuthenticated,] def list(self, request, *args, **kwargs): + """ + List of request.user positions. + This endpoint tries to populate AccountSerializer first using with redis cache. + If redis service was down, it will populate serializer from DB. + """ accounts = read_positions_from_cache(user=request.user) if accounts == 'DB': qs = request.user.accounts.all() diff --git a/readers/tasks.py b/readers/tasks.py index 3b07468..664e50a 100644 --- a/readers/tasks.py +++ b/readers/tasks.py @@ -11,6 +11,10 @@ @shared_task def cache_positions_all_users(): + """ + Caches all Users positions in redis if service is active. + Function implements multi-thread processing to boost speed of caching process. + """ users = User.objects.all() pool = ThreadPool(settings.THREAD_POOL) params = list(users) From e058bd3ce73ce29008e9baa2bea09001bcec00b6 Mon Sep 17 00:00:00 2001 From: alireza51 Date: Wed, 17 May 2023 12:19:25 +0330 Subject: [PATCH 12/12] debug on DB operations --- readers/migrations/0003_alter_account_id.py | 18 ++++++++++++++++++ readers/models.py | 4 ++-- utils/cache.py | 7 ++++++- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 readers/migrations/0003_alter_account_id.py diff --git a/readers/migrations/0003_alter_account_id.py b/readers/migrations/0003_alter_account_id.py new file mode 100644 index 0000000..4bcf60a --- /dev/null +++ b/readers/migrations/0003_alter_account_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-17 08:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('readers', '0002_alter_account_id_field'), + ] + + operations = [ + migrations.AlterField( + model_name='account', + name='id', + field=models.CharField(max_length=512, unique=True), + ), + ] diff --git a/readers/models.py b/readers/models.py index 5cd9591..17495e2 100644 --- a/readers/models.py +++ b/readers/models.py @@ -5,7 +5,7 @@ class Account(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='accounts') - id = models.CharField(max_length=512, null=False, blank=False) + id = models.CharField(max_length=512, null=False, blank=False, unique=True) currency = models.CharField(max_length=10, null=False, blank=False) type = models.CharField(max_length=6, null=False, blank=False) balance = models.DecimalField(max_digits=20, decimal_places=6) @@ -18,4 +18,4 @@ class Meta: unique_together = ('user', 'id',) def __str__(self) -> str: - return f'{self.account_id} - user:{self.user.id}' \ No newline at end of file + return f'{self.id} - user:{self.user.id}' \ No newline at end of file diff --git a/utils/cache.py b/utils/cache.py index fada761..0b3c69c 100644 --- a/utils/cache.py +++ b/utils/cache.py @@ -48,8 +48,13 @@ def write_to_db(user:User, account_list:list) -> None: """ bulk_accounts = [] for item in account_list: + print(item) account = Account(**item) account.user = user bulk_accounts.append(account) # Won't work on Oracle and SQLite < 3.24 because of update_conflicts feature - Account.objects.bulk_create(bulk_accounts, update_conflicts=True, update_fields=['account_type', 'balance', 'available', 'holds']) + Account.objects.bulk_create( + bulk_accounts, update_conflicts=True, + update_fields=['type', 'balance', 'available', 'holds'], + unique_fields=['id'] + )