diff options
| author | Robert Griebl <robert.griebl@qt.io> | 2022-07-29 23:18:37 +0200 |
|---|---|---|
| committer | Robert Griebl <robert.griebl@qt.io> | 2023-08-04 12:59:01 +0000 |
| commit | b3665620377a06f7b7a012e2ae7b69d222fae435 (patch) | |
| tree | 8a012e3ffd952197401f79a396e55e3850fee5ce | |
| parent | 7259d1a839a698e68bc4a7020a63d2aca79a5ec6 (diff) | |
PLEASE NOTE: This project is not maintained anymore. It was ported to
a Qt 6 cmake setup and a more modern Django and Python version to at
least keep it usable for legacy projects.
For non-production use-cases, please switch to the new
appman-package-server available in the Qt Application Manager
starting with version 6.7.
Task-number: AUTOSUITE-1368
Change-Id: Idc4f2490a2a4399c03fce761250f4b5ac2612a45
Reviewed-by: Dominik Holland <dominik.holland@qt.io>
29 files changed, 319 insertions, 450 deletions
diff --git a/.cmake.conf b/.cmake.conf new file mode 100644 index 0000000..56186d7 --- /dev/null +++ b/.cmake.conf @@ -0,0 +1 @@ +set(QT_REPO_MODULE_VERSION "6.7.0") @@ -1,9 +1,3 @@ *~ *.pyc -db.sqlite3 -media/ -static/ -certificates/ -.idea/* -venv/* -static/* +data/* diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d3c8839 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +cmake_minimum_required(VERSION 3.16) + +include(.cmake.conf) +project(QtAutoDeploymentServer + VERSION "${QT_REPO_MODULE_VERSION}" + DESCRIPTION "QtAuto deployment server" + HOMEPAGE_URL "https://qt.io/" + LANGUAGES CXX C +) + +find_package(Qt6 ${PROJECT_VERSION} CONFIG REQUIRED COMPONENTS BuildInternals Core Network) + +add_custom_target(create_docker + COMMAND docker build -t qtauto-deployment-server . + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + VERBATIM +) + +qt_build_repo() + + +if(QT_BUILD_ONLINE_DOCS) + set(DOC_CONF "doc/online/qtautodeploymentserver.qdocconf") +else() + set(DOC_CONF "doc/qtautodeploymentserver.qdocconf") +endif() + +file(GLOB_RECURSE allDocFiles "doc/*.qdoc" "doc/*.png" "doc/*.qdocconf") +add_custom_target(Documentation SOURCES ${allDocFiles}) +qt_internal_add_docs(Documentation ${DOC_CONF}) + +# Add tool dependencies that were deferred by qt_internal_add_docs. +qt_internal_add_deferred_dependencies() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1c0cbc5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ + +FROM debian:bullseye-slim +MAINTAINER Robert Griebl <robert.griebl@qt.io> + +ENV LC_ALL="C.UTF-8" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-pip \ + python3-magic \ + python3-m2crypto + +RUN rm -rf /var/lib/apt/lists/* + +COPY requirements.txt / +RUN pip3 install -r requirements.txt + +RUN mkdir /server +COPY manage.py /server +RUN chmod 755 ./server/manage.py +COPY appstore/ /server/appstore +COPY store/ /server/store + +RUN mkdir /data +VOLUME /data + +ENV APPSTORE_DATA_PATH=/data +ENV APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE /data/certificates/store.p12 +ENV APPSTORE_DEV_VERIFY_CA_CERTIFICATES /data/certificates/ca.crt,/data/certificates/devca.crt + +# You can also set these environment variables: +## ENV APPSTORE_PLATFORM_ID NEPTUNE3 +## ENV APPSTORE_PLATFORM_VERSION 2 +## ENV APPSTORE_DOWNLOAD_EXPIRY 10 +## ENV APPSTORE_BIND_TO_DEVICE_ID 1 +## ENV APPSTORE_NO_SECURITY 1 +## ENV APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE certificates/store.p12 +## ENV APPSTORE_STORE_SIGN_PKCS12_PASSWORD password +## ENV APPSTORE_DEV_VERIFY_CA_CERTIFICATES certificates/ca.crt,certificates/devca.crt + + +COPY docker-entrypoint.sh / +ENTRYPOINT [ "/docker-entrypoint.sh" ] +CMD [ "runserver", "0.0.0.0:8080" ] + +EXPOSE 8080 diff --git a/README.md b/README.md deleted file mode 100644 index 28a473e..0000000 --- a/README.md +++ /dev/null @@ -1,259 +0,0 @@ -This is a PoC deployment server, which can be used together with -the Neptune IVI UI and the Luxoft Application Manager. - -**This is a development server only - do NOT use in production.** - -Architecture -============ - -Setting up the server in virtualenv: - -virtualenv ./venv -./venv/bin/pip install -r requirements.txt - -(libffi-dev is also needed) - - -The server is based on Python/Django. -The reference platform is Debian Jessie and the packages needed there are: - - * python (2.7.9) - * python-yaml (3.11) - * python-django (1.7.9) - * python-django-common (1.7.9) - * python-openssl (0.14) - * python-m2crypto (0.21) - -Before running the server, make sure to adapt the `APPSTORE_*` settings in -`appstore/settings.py` to your environment. - -Since package downloads are done via temporary files, you need to setup -a cron-job to cleanup these temporary files every now and then. The job -should be triggerd every (`settings.APPSTORE_DOWNLOAD_EXPIRY` / 2) minutes -and it just needs to execute: - -``` - ./manage.py expire-downloads -``` - -Commands -======== - -* Running the server: - ``` - ./manage.py runserver 0.0.0.0:8080 - ``` - will start the server on port 8080, reachable for anyone. You can tweak - the listening address to whatever fits your needs. - -* Cleaning up the downloads directory: - ``` - ./manage.py expire-downloads - ``` - will remove all files from the downloads/ directory, that are older than - `settings.APPSTORE_DOWNLOAD_EXPIRY` minutes. - This should be called from a cron-job (see above). - -* Manually verifying a package for upload: - ``` - ./manage.py verify-upload-package <pkg.appkg> - ``` - will tell you if `<pkg.appkg>` is a valid package that can be uploaded to - the store. - -* Manually adding a store signature to a package: - ``` - ./manage.py store-sign-package <in.appkg> <out.appkg> [device id] - ``` - will first verify `<in.appkg>`. If this succeeds, it will copy `<in.appkg>` - to `<out.appkg>` and add a store signature. The optional `[device id]` - parameter will lock the generated package to the device with this id. - -HTTP API -======== - -The deployment server exposes a HTTP API to the world. Arguments to the -functions need to be provided using the HTTP GET syntax. The returned data -will be JSON, PNG or text, depending on the function - -Basic workflow: - -1. Send a `"hello"` request to the server to get the current status and check - whether your platform is compatible with this server instance: - ``` - http://<server>/hello?platform=AM&version=1 - ``` - Returns: - ``` - { "status": "ok" } - ``` - -2. Login as user `'user'` with password `'pass'`: - ``` - http://<server>/login?username=user&password=pass - ``` - Returns: - ``` - { "status": "ok" } - ``` - -3. List all applications - ``` - http://<server>/app/list - ``` - Returns: - ``` - [{ "category": "Entertainment", - "name": "Nice App", - "vendor": "Pelagicore", - "briefDescription": "Nice App is a really nice app.", - "category_id": 4, - "id": "com.pelagicore.niceapp"}, - ... - ] - ``` - -4. Request a download for a App: - ``` - http://<server>/app/purchase?device_id=12345&id=com.pelagicore.niceapp - ``` - Returns: - ``` - { "status": "ok", - "url": "http://<server>/app/download/com.pelagicore.niceapp.2.npkg", - "expiresIn": 600 - } - ``` - -5. Use the `'url'` provided in step 4 to download the application within - `'expiresIn'` seconds. - - -API Reference -============= - -## hello -Checks whether you are using the right Platform and the right API to communicate with the Server. - -| Parameter | Description | -| ---------- | ----------- | -| `platform` | The platform the client is running on, this sets the architecture of the packages you get. (see `settings.APPSTORE_PLATFORM`) | -| `version` | The Deployment Server HTTP API version you are using to communicate with the server. (see `settings.APPSTORE_VERSION`) | -| `require_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. Only applications containing any of specified tags should be listed. Tags must be alphanumeric. | -| `conflicts_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. No application containing any of the tags should be listed. Tags must be alphanumeric. | -| `architecture` | Optional parameter for filtering packages by architecture. Receives cpu architecture. If architecture is not specified, only packages showing 'All' architecture are listed. | - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `maintenance` | The Server is in maintenance mode and can't be used at the moment. | -| | `incompatible-platform` | You are using an incompatible Platform. | -| | `incompatible-version` | You are using an incompatible Version of the API. | -| | `malformed-tag` | Tag had wrong format, was not alphanumeric or could not be parsed. | - -## login -Does a login on the Server with the given username and password. Either a imei or a mac must be provided. This call is needed for downloading apps. - -| Parameter | Description | -| ---------- | ----------- | -| `username` | The username. | -| `password` | The password for the given username | - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `missing-credentials` | Forgot to provided username and/or password. | -| | `account-disabled` | The account is disabled. | -| | `authentication-failed` | Failed to authenticate with given username and password. | - - -## logout -Does a logout on the Server for the currently logged in user. - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `failed` | Not logged in. | - -## app/list -Lists all apps. The returned List can be filtered by using the category_id and the filter argument. - -| Parameter | Description | -| ------------- | ----------- | -| `category_id` | Only lists apps, which are in the category with this id. | -| `filter` | Only lists apps, whose name matches the filter. | - -Returns a JSON array (not an object!). Each field is a JSON object: - -| JSON field | Description | -| ------------------ | ----------- | -| `id` | The unique id of the application | -| `name` | The name of the application | -| `vendor` | The name of the vendor of this application -| `briedDescription` | A short (one line) description of the application -| `category` | The name of the category the application is in -| `category_id` | The id of the category the application is in - -## app/icon - Returns an icon for the given application id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The application id | - - Returns a PNG image or a 404 error - - -## app/description -Returns a description for the given application id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The application id | - -Returns text - either HTML or plain - - -## app/purchase -Returns an url which can be used for downloading the requested application for -certain period of time (configurable in the settings) - -| Parameter | Description | -| ----------- | ----------- | -| `device_id` | The unique device id of the client hardware. | -| `id` | The application Id. | - -Returns a JSON object: - -| JSON field | Value | Description | -| ----------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `failed` | Something went wrong. See the `error` field for more information. | -| `error` | **text** | An error description, if `status` is `failed. | -| `url` | **url** | The url which can now be used for downloading the application. | -| `expiresIn` | **int** | The time in seconds the url remains valid. | - -## category/list -Lists all the available categories. It uses the rank stored on the server for ordering. - -Returns a JSON array (not an object!). Each field is a JSON object: - -| JSON field | Description | -| ---------- | ----------- | -| `id` | The unique id of the category | -| `name` | The name of the category | - -## category/icon: -Returns an icon for the given category id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The id of the category | - -Returns a PNG image or a 404 error diff --git a/appstore/settings.py b/appstore/settings.py index b03e943..bdd6e44 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -41,17 +41,16 @@ https://docs.djangoproject.com/en/1.7/ref/settings/ """ import os -APPSTORE_MAINTENANCE = False -APPSTORE_PLATFORM_ID = 'NEPTUNE3' -APPSTORE_PLATFORM_VERSION = 2 # Maximum supported platform version: - # version 1 - only old package format - # version 2 - old and new package formats -APPSTORE_DOWNLOAD_EXPIRY = 10 # in minutes -APPSTORE_BIND_TO_DEVICE_ID = True # unique downloads for each device -APPSTORE_NO_SECURITY = True # ignore developer signatures and do not generate store signatures -APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE = 'certificates/store.p12' -APPSTORE_STORE_SIGN_PKCS12_PASSWORD = 'password' -APPSTORE_DEV_VERIFY_CA_CERTIFICATES = ['certificates/ca.crt', 'certificates/devca.crt'] +APPSTORE_MAINTENANCE = False +APPSTORE_PLATFORM_ID = os.getenv('APPSTORE_PLATFORM_ID', default = 'NEPTUNE3') +APPSTORE_PLATFORM_VERSION = int(os.getenv('APPSTORE_PLATFORM_VERSION', default = '2')) +APPSTORE_DOWNLOAD_EXPIRY = int(os.getenv('APPSTORE_DOWNLOAD_EXPIRY', default = '10')) # in minutes +APPSTORE_BIND_TO_DEVICE_ID = os.getenv('APPSTORE_BIND_TO_DEVICE_ID', default = '1') == '1' # unique downloads for each device +APPSTORE_NO_SECURITY = os.getenv('APPSTORE_NO_SECURITY', default = '1') == '1' # ignore developer signatures and do not generate store signatures +APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE = os.getenv('APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE', default = 'certificates/store.p12') +APPSTORE_STORE_SIGN_PKCS12_PASSWORD = os.getenv('APPSTORE_STORE_SIGN_PKCS12_PASSWORD', default = 'password') +APPSTORE_DEV_VERIFY_CA_CERTIFICATES = os.getenv('APPSTORE_DEV_VERIFY_CA_CERTIFICATES', ','.join(['certificates/ca.crt', 'certificates/devca.crt'])).split(',') +APPSTORE_DATA_PATH = os.getenv('APPSTORE_DATA_PATH', default = '') # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -61,12 +60,12 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__)) # See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '4%(o_1zuz@^kjcarw&!5ptvk	oa1-83*arn6jcm4idzy1#30' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', default = '4%(o_1zuz@^kjcarw&!5ptvk	oa1-83*arn6jcm4idzy1#30') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*'] TEMPLATES = [ { @@ -77,6 +76,7 @@ TEMPLATES = [ 'context_processors': ( "django.contrib.auth.context_processors.auth", "django.template.context_processors.debug", + "django.template.context_processors.request", "django.template.context_processors.i18n", "django.template.context_processors.media", "django.template.context_processors.static", @@ -102,12 +102,11 @@ INSTALLED_APPS = ( 'store', ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ) @@ -116,6 +115,12 @@ ROOT_URLCONF = 'appstore.urls' WSGI_APPLICATION = 'appstore.wsgi.application' +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +if not APPSTORE_DATA_PATH: + MEDIA_ROOT = os.path.join(BASE_DIR, 'data/') +else: + MEDIA_ROOT = APPSTORE_DATA_PATH # Database # https://docs.djangoproject.com/en/1.7/ref/settings/#databases @@ -123,16 +128,16 @@ WSGI_APPLICATION = 'appstore.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } } # Internationalization # https://docs.djangoproject.com/en/1.7/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = os.getenv('DJANGO_LANGUAGE_CODE', default = 'en-us') -TIME_ZONE = 'Europe/Berlin' +TIME_ZONE = os.getenv('DJANGO_TIME_ZONE', default = 'Europe/Berlin') USE_I18N = True @@ -148,9 +153,6 @@ URL_PREFIX = '' # Shouldn't start with '/' in case it is used STATIC_URL = '/static/' -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(BASE_DIR, 'media/') STATIC_ROOT = os.path.join(BASE_DIR, 'static') # URL that handles the media served from MEDIA_ROOT. Make sure to use a @@ -164,3 +166,4 @@ ICON_SIZE_Y = 50 # If the icon should be transformed to monochrome, with alpha channel, when uploaded or not ICON_DECOLOR = False +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/appstore/urls.py b/appstore/urls.py index 1eb3d2c..667b899 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -30,26 +30,26 @@ ## ############################################################################# -from django.conf.urls import include, url +from django.urls import include, re_path from django.contrib import admin from store import api as store_api from appstore.settings import URL_PREFIX base_urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + re_path(r'^admin/', admin.site.urls), - url(r'^hello$', store_api.hello), - url(r'^login$', store_api.login), - url(r'^logout$', store_api.logout), - url(r'^app/list$', store_api.appList), - url(r'^app/icons/(.*)$', store_api.appIconNew), - url(r'^app/icon', store_api.appIcon), - url(r'^app/description', store_api.appDescription), - url(r'^app/purchase', store_api.appPurchase), - url(r'^app/download/(.*)$', store_api.appDownload), - url(r'^category/list$', store_api.categoryList), - url(r'^category/icon$', store_api.categoryIcon), - url(r'^upload$', store_api.upload), + re_path(r'^hello$', store_api.hello), + re_path(r'^login$', store_api.login), + re_path(r'^logout$', store_api.logout), + re_path(r'^app/list$', store_api.appList), + re_path(r'^app/icon$', store_api.appIcon), + re_path(r'^app/icons/(.*)$', store_api.appIconNew), + re_path(r'^app/description', store_api.appDescription), + re_path(r'^app/purchase', store_api.appPurchase), + re_path(r'^app/download/(.*)$', store_api.appDownload), + re_path(r'^category/list$', store_api.categoryList), + re_path(r'^category/icon$', store_api.categoryIcon), + re_path(r'^upload$', store_api.upload), ] @@ -58,6 +58,6 @@ if URL_PREFIX != '': prefix = prefix + URL_PREFIX + '/' urlpatterns = [ - url(prefix, include(base_urlpatterns)), + re_path(prefix, include(base_urlpatterns)), ] diff --git a/doc/QtAutoDeploymentServerDoc b/doc/QtAutoDeploymentServerDoc new file mode 100644 index 0000000..1c92d7c --- /dev/null +++ b/doc/QtAutoDeploymentServerDoc @@ -0,0 +1,3 @@ +// needed for the new clang based qdoc parser in Qt 5.11 +#include <QtAppManMain/QtAppManMain> +#include <QtCore/QtCore> diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf index 258ef99..d310e7a 100644 --- a/doc/qtautodeploymentserver-project.qdocconf +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -3,9 +3,14 @@ description = Qt Automotive Suite Deployment Server Documentat version = $QT_VERSION url = https://doc.qt.io/QtAutoDeploymentServer +# needed for the new clang based qdoc parser +moduleheader = QtAutoDeploymentServerDoc +includepaths = -I. + sourcedirs += src imagedirs += src/images +depends += qtcore qtapplicationmanager qhp.projects = QtAutoDeploymentServer qhp.QtAutoDeploymentServer.file = qtautodeploymentserver.qhp @@ -20,8 +25,8 @@ qhp.QtAutoDeploymentServer.customFilters.Qt.filterAttributes = qtautodeployments tagfile = qtautodeploymentserver.tags -depends = qtautomotivesuite - -buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" -navigation.homepage = "Qt Automotive Suite" navigation.landingpage = "Qt Automotive Suite Deployment Server" +buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" + +# Fail the documentation build if there are more warnings than the limit +warninglimit = 0 diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index b21ee76..d31b72c 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -27,13 +27,12 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-http-server-setup.html + \ingroup qtauto-deployment-server + \page http-server-setup.html \previouspage Qt Automotive Suite Deployment Server API Reference \nextpage Upload Packages to the Deployment Server - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server - \title Set up a Production server with Apache, Lighttpd or Nginx The Deployment Server can be set up in combination with a regular web server: Apache, Lighttpd, or diff --git a/doc/src/deployment-server-installation.qdoc b/doc/src/deployment-server-installation.qdoc index 7937992..06fb895 100644 --- a/doc/src/deployment-server-installation.qdoc +++ b/doc/src/deployment-server-installation.qdoc @@ -27,14 +27,24 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-installation.html - \contentspage {Qt Automotive Suite Deployment Server} + \ingroup qtauto-deployment-server + \page installation.html \previouspage Qt Automotive Suite Deployment Server \nextpage Qt Automotive Suite Deployment Server API Reference \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server Installation + \section1 Set up the Server in a Docker Container + + The new recommended way to run this server is through the supplied \c Dockerfile and the + \c docker-manage.sh script, which can both be found in the modules root directory. + + Instead of messing with Django's project configuration, you can simply export your individual + settings as environment variables. Either directly in the \c Dockerfile when building the + container, or by copying and modifying the \c docker-manage.sh script. + + \section1 Set up the Server in a Virtual Environment Before you install the dependencies in the Python virtual environment, you need to install the diff --git a/doc/src/deployment-server-package-upload.qdoc b/doc/src/deployment-server-package-upload.qdoc index dc5b7b4..f2e09de 100644 --- a/doc/src/deployment-server-package-upload.qdoc +++ b/doc/src/deployment-server-package-upload.qdoc @@ -27,10 +27,10 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-package-upload.html + \ingroup qtauto-deployment-server + \page package-upload.html \previouspage Set up a Production server with Apache, Lighttpd or Nginx \nextpage Qt Automotive Suite Deployment Server - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server @@ -43,7 +43,7 @@ server itself. \section1 Through Server Admin Page This was the first uploading method implemented. It uses django admin page, accessible by \c{/admin/} URL of -the Deployment Server. For Qt 5.14, the URL is \l{http://demoappsdeploy.qt.io:8514/admin/}. +the Deployment Server. To add application: \list diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index d4be3af..6036e67 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -27,10 +27,10 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-reference.html + \ingroup qtauto-deployment-server + \page reference.html \previouspage Qt Automotive Suite Deployment Server Installation \nextpage Set up a Production server with Apache, Lighttpd or Nginx - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server API Reference diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 50f2bee..9c6390c 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -27,13 +27,19 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-index.html - \contentspage {Qt Automotive Suite} + \ingroup qtauto-deployment-server + \page index.html \nextpage Qt Automotive Suite Deployment Server Installation \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server + \note This project is not maintained anymore. It was ported to a Qt 6 cmake setup and a more + modern Django and Python version to at least keep it usable for legacy projects. + For non-production use-cases, please switch to the new + \l{Package-Server}{appman-package-server} + available in the \l{Qt Application Manager} starting with version 6.7. + The Qt Automotive Suite Deployment Server is a new component in the Qt Automotive Suite 5.12. Previously, it was known as the Neptune Appstore and used for demonstrations purposes. diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..6e7fde5 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd /server +exec ./manage.py "$@" diff --git a/docker-manage.sh b/docker-manage.sh new file mode 100755 index 0000000..e1cef29 --- /dev/null +++ b/docker-manage.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +#export APPSTORE_PLATFORM_ID NEPTUNE3 +#export APPSTORE_PLATFORM_VERSION 2 +#export APPSTORE_DOWNLOAD_EXPIRY 10 +#export APPSTORE_BIND_TO_DEVICE_ID 1 +#export APPSTORE_NO_SECURITY 1 +#export APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE certificates/store.p12 +#export APPSTORE_STORE_SIGN_PKCS12_PASSWORD password +#export APPSTORE_DEV_VERIFY_CA_CERTIFICATES certificates/ca.crt,certificates/devca.crt + +IT="" +if [ "x$1" = "x-it" ]; then + shift + IT=-it +fi + +cd `dirname $0`/.. +mkdir -p data + +exec docker run $IT \ + -p 8080:8080 \ + -v `pwd`/data:/data \ + -e APPSTORE_PLATFORM_ID \ + -e APPSTORE_PLATFORM_VERSION \ + -e APPSTORE_DOWNLOAD_EXPIRY \ + -e APPSTORE_BIND_TO_DEVICE_ID \ + -e APPSTORE_NO_SECURITY \ + -e APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE \ + -e APPSTORE_STORE_SIGN_PKCS12_PASSWORD \ + -e APPSTORE_DEV_VERIFY_CA_CERTIFICATES \ + qtauto-deployment-server "$@" @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 ############################################################################# ## ## Copyright (C) 2019 Luxoft Sweden AB diff --git a/qtauto-deployment-server.pro b/qtauto-deployment-server.pro deleted file mode 100644 index f7f9002..0000000 --- a/qtauto-deployment-server.pro +++ /dev/null @@ -1,13 +0,0 @@ -TEMPLATE = aux - -build_online_docs: { - QMAKE_DOCS = $$PWD/doc/online/qtautodeploymentserver.qdocconf -} else { - QMAKE_DOCS = $$PWD/doc/qtautodeploymentserver.qdocconf -} - -OTHER_FILES += \ - $$PWD/doc/*.qdocconf \ - $$PWD/doc/src/*.qdoc - -load(qt_docs) diff --git a/requirements.txt b/requirements.txt index 92c71f9..4693615 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYAML -django==1.11.27 +django==4.0.6 django-common -django-ordered-model==2.1 +django-ordered-model pyOpenSSL M2Crypto Enum34 @@ -10,4 +10,4 @@ cffi paramiko cryptography pillow -python-magic==0.4.15 +python-magic diff --git a/store/admin.py b/store/admin.py index 910f3fb..389bb87 100644 --- a/store/admin.py +++ b/store/admin.py @@ -30,15 +30,14 @@ ## ############################################################################# -import StringIO +import io from PIL import Image, ImageChops from django import forms from django.contrib import admin -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy from django.core.files.uploadedfile import InMemoryUploadedFile from ordered_model.admin import OrderedModelAdmin +from django.utils.safestring import mark_safe from store.models import * from store.utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList @@ -68,10 +67,11 @@ class CategoryAdminForm(forms.ModelForm): im = Image.open(cleaned_data['icon']) size = (settings.ICON_SIZE_X, settings.ICON_SIZE_Y) im.thumbnail(size, Image.ANTIALIAS) - imagefile = StringIO.StringIO() + imagefile = io.BytesIO() im.save(imagefile, format='png') + length = imagefile.tell() imagefile.seek(0) - cleaned_data['icon'] = InMemoryUploadedFile(imagefile, 'icon', "icon.png", 'image/png', imagefile.len, None) + cleaned_data['icon'] = InMemoryUploadedFile(imagefile, 'icon', "icon.png", 'image/png', length, None) return cleaned_data class CategoryAdmin(OrderedModelAdmin): @@ -85,16 +85,15 @@ class CategoryAdmin(OrderedModelAdmin): def name(self, obj): # just to forbid sorting by name return obj.name - name.short_description = ugettext_lazy('Item caption') + name.short_description = u'Item caption' def icon_image(self, obj): prefix = settings.URL_PREFIX image_request = prefix + "/category/icon?id=%s" % (obj.id) - html = u'<img width=%s height=%s src="%s" />' % (settings.ICON_SIZE_X, settings.ICON_SIZE_Y, image_request) - return html + html = '<img width=%s height=%s src="%s" />' % (settings.ICON_SIZE_X, settings.ICON_SIZE_Y, image_request) + return mark_safe(html) - icon_image.allow_tags = True - icon_image.short_description = ugettext_lazy('Category icon') + icon_image.short_description = u'Category icon' class AppAdminForm(forms.ModelForm): @@ -113,7 +112,7 @@ class AppAdminForm(forms.ModelForm): try: pkgdata = parseAndValidatePackageMetadata(package_file) except Exception as error: - raise forms.ValidationError(_('Validation error: %s' % str(error))) + raise forms.ValidationError('Validation error: %s' % str(error)) self.appId = pkgdata['info']['id'] self.name = pkgdata['storeName'] @@ -123,27 +122,27 @@ class AppAdminForm(forms.ModelForm): # check if this really is an update if hasattr(self, 'instance') and self.instance.appid: if self.appId != self.instance.appid: - raise forms.ValidationError(_('Validation error: an update cannot change the ' - 'application id, tried to change from %s to %s' % - (self.instance.appid, self.appId))) + raise forms.ValidationError('Validation error: an update cannot change the ' + 'application id, tried to change from %s to %s' % + (self.instance.appid, self.appId)) elif self.architecture != self.instance.architecture: - raise forms.ValidationError(_('Validation error: an update cannot change the ' - 'application architecture from %s to %s' % - (self.instance.architecture, self.architecture))) + raise forms.ValidationError('Validation error: an update cannot change the ' + 'application architecture from %s to %s' % + (self.instance.architecture, self.architecture)) else: try: if App.objects.get(appid__exact=self.appId, architecture__exact=self.architecture, tags_hash__exact=self.tags_hash): - raise forms.ValidationError(_('Validation error: another application with id' - ' %s , tags %s and architecture %s already ' - 'exists' % (str(self.appId), str(self.tags_hash), - str(self.architecture)))) + raise forms.ValidationError('Validation error: another application with id' + ' %s , tags %s and architecture %s already ' + 'exists' % (str(self.appId), str(self.tags_hash), + str(self.architecture))) except App.DoesNotExist: pass # write icon into file to serve statically success, error = writeTempIcon(self.appId, self.architecture, self.tags_hash, pkgdata['icon']) if not success: - raise forms.ValidationError(_(error)) + raise forms.ValidationError(error) return cleaned_data diff --git a/store/api.py b/store/api.py index 9444b0b..b617e34 100644 --- a/store/api.py +++ b/store/api.py @@ -33,6 +33,7 @@ import os import shutil import hashlib +import logging from django.conf import settings from django.db.models import Q, Count @@ -89,7 +90,6 @@ def hello(request): else: request.session['architecture'] = '' - request.session['pkgversions'] = range(1, version + 1) return JsonResponse({'status': status}) @@ -120,7 +120,7 @@ def login(request): def logout(request): status = 'ok' - if not request.user.is_authenticated(): + if not request.user.is_authenticated: status = 'failed' logout(request) @@ -201,12 +201,7 @@ def appList(request): if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] - apps = apps.filter(architecture__in=archlist) - apps = apps.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed @@ -261,13 +256,9 @@ def appDescription(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] appId = getRequestDictionary(request)['id'] try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') - app = app.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: @@ -285,7 +276,7 @@ def appIconNew(request, path): path=path.replace('/', '_').replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' try: response = HttpResponse(content_type='image/png') - with open(iconPath() + path, 'rb') as pkg: + with open(os.path.join(settings.MEDIA_ROOT, iconPath(), path), 'rb') as pkg: response.write(pkg.read()) response['Content-Length'] = pkg.tell() return response @@ -299,22 +290,20 @@ def appIcon(request): archlist.append(normalizeArch(dictionary['architecture'])) elif 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] appId = dictionary['id'] try: - app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') - app = app.filter(pkgformat__in=versionlist) + apps = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: tags = SoftwareTagList() tags.parse(request.session['tag']) - app_ids = [x.id for x in app if x.is_tagmatching(tags.list())] - app = App.objects.filter(id__in=app_ids) - app = app.last() - with open(iconPath(app.appid, app.architecture, app.tags_hash), 'rb') as iconPng: + app_ids = [x.id for x in apps if x.is_tagmatching(tags.list())] + apps = App.objects.filter(id__in=app_ids) + app = apps.last() + path = iconPath(app.appid, app.architecture, app.tags_hash) + path = os.path.join(settings.MEDIA_ROOT, path) + with open(path, 'rb') as iconPng: response = HttpResponse(content_type='image/png') response.write(iconPng.read()) return response @@ -323,14 +312,11 @@ def appIcon(request): def appPurchase(request): - if not request.user.is_authenticated(): + if not request.user.is_authenticated: return HttpResponseForbidden('no login') archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] try: deviceId = str(getRequestDictionary(request).get("device_id", "")) @@ -346,7 +332,6 @@ def appPurchase(request): app = App.objects.filter(id__exact = getRequestDictionary(request)['purchaseId'], architecture__in=archlist).order_by('architecture','tags_hash') else: raise ValidationError('id or purchaseId parameter required') - app = app.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: @@ -356,7 +341,7 @@ def appPurchase(request): app = App.objects.filter(id__in=app_ids) app = app.last() - fromFilePath = packagePath(app.appid, app.architecture, app.tags_hash) + fromFilePath = os.path.join(settings.MEDIA_ROOT, packagePath(app.appid, app.architecture, app.tags_hash)) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) @@ -423,7 +408,7 @@ def categoryIcon(request): try: if categoryId != '-1': category = Category.objects.filter(id__exact = categoryId)[0] - filename = iconPath() + "category_" + str(category.id) + ".png" + filename = os.path.join(settings.MEDIA_ROOT, iconPath(), "category_" + str(category.id) + ".png") else: from django.contrib.staticfiles import finders filename = finders.find('img/category_All.png') @@ -439,7 +424,7 @@ def categoryIcon(request): # |Error| # | | # +-----+ - emptyPng = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x01\x03\x00\x00\x00I\xb4\xe8\xb7\x00\x00\x00\x06PLTE\x00\x00\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\x33IDAT\x08\xd7\x63\xf8\x0f\x04\x0c\x0d\x0c\x0c\x8c\x44\x13\x7f\x40\xc4\x01\x10\x71\xb0\xf4\x5c\x2c\xc3\xcf\x36\xc1\x44\x86\x83\x2c\x82\x8e\x48\xc4\x5f\x16\x3e\x47\xd2\x0c\xc5\x46\x80\x9c\x06\x00\xa4\xe5\x1d\xb4\x8e\xae\xe8\x43\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82" + emptyPng = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x01\x03\x00\x00\x00I\xb4\xe8\xb7\x00\x00\x00\x06PLTE\x00\x00\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\x33IDAT\x08\xd7\x63\xf8\x0f\x04\x0c\x0d\x0c\x0c\x8c\x44\x13\x7f\x40\xc4\x01\x10\x71\xb0\xf4\x5c\x2c\xc3\xcf\x36\xc1\x44\x86\x83\x2c\x82\x8e\x48\xc4\x5f\x16\x3e\x47\xd2\x0c\xc5\x46\x80\x9c\x06\x00\xa4\xe5\x1d\xb4\x8e\xae\xe8\x43\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82" response.write(emptyPng) return response diff --git a/store/authdecorators.py b/store/authdecorators.py index 2a4119c..a4307ff 100644 --- a/store/authdecorators.py +++ b/store/authdecorators.py @@ -60,7 +60,7 @@ def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): # NOTE: We are only support basic authentication for now. # if auth[0].lower() == "basic": - uname, passwd = base64.b64decode(auth[1]).split(':') + uname, passwd = base64.b64decode(auth[1].encode('utf-8')).decode('utf-8').split(':') user = authenticate(username=uname, password=passwd) if user is not None: if user.is_active: @@ -114,7 +114,7 @@ def logged_in_or_basicauth(realm=""): def view_decorator(func): def wrapper(request, *args, **kwargs): return view_or_basicauth(func, request, - lambda u: u.is_authenticated(), + lambda u: u.is_authenticated, realm, *args, **kwargs) return wrapper diff --git a/store/management/commands/expire-downloads.py b/store/management/commands/expire-downloads.py index 94b0d24..012182e 100644 --- a/store/management/commands/expire-downloads.py +++ b/store/management/commands/expire-downloads.py @@ -39,7 +39,7 @@ from django.conf import settings from store.utilities import downloadPath class Command(BaseCommand): - help = 'Expires all downloads that are older than 10 minutes' + help = 'Expires all downloads that are older than APPSTORE_DOWNLOAD_EXPIRY minutes' def handle(self, *args, **options): self.stdout.write('Removing expired download packages') @@ -50,7 +50,7 @@ class Command(BaseCommand): for pkg in os.listdir(pkgPath): t = os.path.getmtime(pkgPath + pkg) age = time.time() - t - if age > (10 * 60): + if age > (int(settings.APPSTORE_DOWNLOAD_EXPIRY) * 60): os.remove(pkgPath + pkg) self.stdout.write(' -> %s (age: %s seconds)' % (pkg, int(age))) diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 80d154c..2bb3457 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.27 on 2020-08-14 17:24 ############################################################################# ## ## Copyright (C) 2020 Luxoft Sweden AB @@ -32,7 +31,7 @@ ## ############################################################################# -# Generated by Django 1.11.27 on 2020-07-22 16:39 +# Generated by Django 1.11.27 on 2020-10-30 13:46 from __future__ import unicode_literals from django.conf import settings @@ -62,9 +61,9 @@ class Migration(migrations.Migration): ('description', models.TextField()), ('dateAdded', models.DateField(auto_now_add=True)), ('dateModified', models.DateField(auto_now=True)), - ('tags_hash', models.CharField(default=b'', max_length=4096)), - ('architecture', models.CharField(default=b'All', max_length=20)), - ('version', models.CharField(default=b'0.0.0', max_length=20)), + ('tags_hash', models.CharField(default='', max_length=4096)), + ('architecture', models.CharField(default='All', max_length=20)), + ('version', models.CharField(default='0.0.0', max_length=20)), ('pkgformat', models.IntegerField()), ], ), diff --git a/store/migrations/0002_alter_category_order.py b/store/migrations/0002_alter_category_order.py new file mode 100644 index 0000000..e13175c --- /dev/null +++ b/store/migrations/0002_alter_category_order.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.6 on 2023-06-26 19:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('store', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='category', + name='order', + field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + ), + ] diff --git a/store/models.py b/store/models.py index 7e8751a..ff8ab0c 100644 --- a/store/models.py +++ b/store/models.py @@ -45,7 +45,7 @@ from store.tags import SoftwareTag def category_file_name(instance, filename): # filename parameter is unused. See django documentation for details: # https://docs.djangoproject.com/en/1.11/ref/models/fields/#django.db.models.FileField.upload_to - return settings.MEDIA_ROOT + "icons/category_" + str(instance.id) + ".png" + return "icons/category_" + str(instance.id) + ".png" class OverwriteStorage(FileSystemStorage): def get_available_name(self, name, max_length=None): @@ -63,6 +63,9 @@ class Category(OrderedModel): def __unicode__(self): return self.name + def __str__(self): + return self.name + def save(self, *args, **kwargs): if self.id is None: # This is a django hack. When category icon is saved and then later accessed, @@ -77,13 +80,16 @@ class Category(OrderedModel): super(Category, self).save(*args, **kwargs) class Vendor(models.Model): - user = models.ForeignKey(User, primary_key = False) + user = models.ForeignKey(User, primary_key = False, on_delete = models.CASCADE) name = models.CharField(max_length = 200) certificate = models.TextField(max_length = 8000) def __unicode__(self): return self.name + def __str__(self): + return self.name + class Tag(models.Model): negative = models.BooleanField(default=False) name = models.CharField(max_length=200) @@ -106,8 +112,8 @@ class App(models.Model): appid = models.CharField(max_length=200) name = models.CharField(max_length=200) file = models.FileField(upload_to=content_file_name, storage=OverwriteStorage()) - vendor = models.ForeignKey(Vendor) - category = models.ForeignKey(Category) + vendor = models.ForeignKey(Vendor, on_delete = models.CASCADE) + category = models.ForeignKey(Category, on_delete = models.CASCADE) briefDescription = models.TextField() description = models.TextField() dateAdded = models.DateField(auto_now_add=True) diff --git a/store/tags.py b/store/tags.py index e906684..da09803 100644 --- a/store/tags.py +++ b/store/tags.py @@ -79,7 +79,7 @@ class SoftwareTag: """ Takes tag and parses it. If it can't parse - raises exception of invalid value :type tag: str """ - if not isinstance(tag, (str, unicode)): + if not isinstance(tag, str): raise BaseException("Invalid input data-type") if not validateTag(tag): raise BaseException("Malformed tag") @@ -121,7 +121,7 @@ class SoftwareTagList: def __str__(self): lst = list() - for _, value in self.taglist.items(): + for _, value in list(self.taglist.items()): lst += [str(i) for i in value] lst.sort() return ",".join(lst) @@ -168,7 +168,7 @@ class SoftwareTagList: def list(self): lst = list() - for _, value in self.taglist.items(): + for _, value in list(self.taglist.items()): for i in value: lst.append(i) return lst @@ -188,13 +188,13 @@ class TestSoftwareTagMethods(unittest.TestCase): self.assertTrue(tag.has_version()) tag = SoftwareTag('Qt') self.assertFalse(tag.has_version()) - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('фыва') - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('фыва:5.1') - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('qt.1:5.1') - with self.assertRaisesRegexp(BaseException, "Invalid input data-type"): + with self.assertRaisesRegex(BaseException, "Invalid input data-type"): SoftwareTag(1) def test_tag_match(self): @@ -227,11 +227,11 @@ class TestSoftwareTagListMethods(unittest.TestCase): def test_append_invalid(self): lst = SoftwareTagList() - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag('qt:1:1'))) # Invalid version - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag('фыва'))) # Non-ascii - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag(''))) # empty tag is not valid def test_append_valid(self): diff --git a/store/utilities.py b/store/utilities.py index 06151bb..1a283e5 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -70,28 +70,29 @@ def getRequestDictionary(request): return request.GET def packagePath(appId=None, architecture=None, tags=None): - path = settings.MEDIA_ROOT + 'packages/' + path = "packages" ## os.path.join(settings.MEDIA_ROOT, 'packages/') if tags is None: tags = "" if (appId is not None) and (architecture is not None): - path = path + '_'.join([appId, architecture, tags]).replace('/', '_').\ - replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + path = os.path.join(path, '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C')) return path def iconPath(appId=None, architecture=None, tags=None): - path = settings.MEDIA_ROOT + 'icons/' + path = "icons" ## os.path.join(settings.MEDIA_ROOT, 'icons/') if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/', '_').\ - replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' + return os.path.join(path, '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png') return path def writeTempIcon(appId, architecture, tags, icon): try: - if not os.path.exists(iconPath()): - os.makedirs(iconPath()) - tempicon = open(iconPath(appId, architecture, tags), 'w') + path = os.path.join(settings.MEDIA_ROOT, iconPath()) + if not os.path.exists(path): + os.makedirs(path) + tempicon = open(os.path.join(settings.MEDIA_ROOT, iconPath(appId, architecture, tags)), 'wb') tempicon.write(icon) tempicon.flush() tempicon.close() @@ -101,39 +102,35 @@ def writeTempIcon(appId, architecture, tags, icon): str(error) def downloadPath(): - return settings.MEDIA_ROOT + 'downloads/' + return os.path.join(settings.MEDIA_ROOT, 'downloads/') -def isValidDnsName(dnsName, errorList): - # see also in AM: src/common-lib/utilities.cpp / isValidDnsName() +def isValidFilesystemName(name, errorList): + # see also in AM: src/common-lib/utilities.cpp / validateForFilesystemUsage try: - # this is not based on any RFC, but we want to make sure that this id is usable as filesystem - # name. So in order to support FAT (SD-Cards), we need to keep the path < 256 characters + # we need to make sure that we can use the name as directory in a filesystem and inode names + # are limited to 255 characters in Linux. We need to subtract a safety margin for prefixes + # or suffixes though: - if len(dnsName) > 200: - raise Exception('too long - the maximum length is 200 characters') + if not name: + raise Exception('must not be empty') - # we require at least 3 parts: tld.company-name.application-name - # this make it easier for humans to identify apps by id. + if len(name) > 150: + raise Exception('the maximum length is 150 characters') - labels = dnsName.split('.') - if len(labels) < 3: - raise Exception('wrong format - needs to be in reverse-DNS notation and consist of at least three parts separated by .') + # all characters need to be ASCII minus any filesystem special characters: + spaceOnly = True + forbiddenChars = '<>:"/\\|?*' + for i, c in enumerate(name): + if (ord(c) < 0x20) or (ord(c) > 0x7f) or (c in forbiddenChars): + raise Exception(f'must consist of printable ASCII characters only, except any of \'{forbiddenChars}\'') - # standard domain name requirements from the RFCs 1035 and 1123 + if spaceOnly: + spaceOnly = (c == ' ') - for label in labels: - if 0 >= len(label) > 63: - raise Exception('wrong format - each part of the name needs to at least 1 and at most 63 characters') - - for i, c in enumerate(label): - isAlpha = (c >= '0' and c <= '9') or (c >= 'a' and c <= 'z'); - isDash = (c == '-'); - isInMiddle = (i > 0) and (i < (len(label) - 1)); - - if not (isAlpha or (isDash and isInMiddle)): - raise Exception('invalid characters - only [a-z0-9-] are allowed (and '-' cannot be the first or last character)') + if spaceOnly: + raise Exception('must not consist of only white-space characters') return True @@ -141,7 +138,6 @@ def isValidDnsName(dnsName, errorList): errorList[0] = str(error) return False - def verifySignature(signaturePkcs7, hash, chainOfTrust): # see also in AM: src/crypto-lib/signature.cpp / Signature::verify() @@ -256,7 +252,7 @@ def parsePackageMetadata(packageFile): raise Exception('the first file in the package is not --PACKAGE-HEADER--, but %s' % entry.name) if entry.name.startswith('--PACKAGE-FOOTER--'): - footerContents += contents + footerContents += contents.decode('utf-8') foundFooter = True elif foundFooter: @@ -264,10 +260,11 @@ def parsePackageMetadata(packageFile): if not entry.name.startswith('--PACKAGE-'): addToDigest1 = '%s/%s/' % ('D' if entry.isdir() else 'F', 0 if entry.isdir() else entry.size) + addToDigest1 = addToDigest1.encode('utf-8') entryName = entry.name if entry.isdir() and entryName.endswith('/'): entryName = entryName[:-1] - addToDigest2 = unicode(entryName, 'utf-8').encode('utf-8') + addToDigest2 = str(entryName).encode('utf-8') if entry.isfile(): digest.update(contents) @@ -380,7 +377,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): 'icon': [], 'digest': [] } - for part in partFields.keys(): + for part in list(partFields.keys()): if not part in pkgdata: raise Exception('package metadata is missing the %s part' % part) data = pkgdata[part] @@ -393,7 +390,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header'][packageIdKey], pkgdata['info']['id'])) error = [''] - if not isValidDnsName(pkgdata['info']['id'], error): + if not isValidFilesystemName(pkgdata['info']['id'], error): raise Exception('invalid id: %s' % error[0]) if pkgdata['header']['diskSpaceUsed'] <= 0: @@ -408,7 +405,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): elif 'en_US' in pkgdata['info']['name']: name = pkgdata['info']['name']['en_US'] elif len(pkgdata['info']['name']) > 0: - name = pkgdata['info']['name'].values()[0] + name = list(pkgdata['info']['name'].values())[0] if not name: raise Exception('could not deduce a suitable package name from the info part') @@ -456,7 +453,7 @@ def addFileToPackage(sourcePackageFile, destinationPackageFile, fileName, fileCo entry = dst.gettarinfo(fileobj = tmp, arcname = fileName) entry.uid = entry.gid = 0 entry.uname = entry.gname = '' - entry.mode = 0400 + entry.mode = 0o400 dst.addfile(entry, fileobj = tmp) dst.close() diff --git a/tests/tests.pro b/tests/tests.pro deleted file mode 100644 index 9671085..0000000 --- a/tests/tests.pro +++ /dev/null @@ -1 +0,0 @@ -TEMPLATE = subdirs |