Refactor database initialisation

Changes in this commit include:

* Quality checks now run on a in-memory SQLite database so they don't
mess around with other test data used in the development environment.

* That in-memory database is populated automatically in a pytest
fixture, so there's no need any more to call "initdb" before running
back's pytest.

* There's a new script fsfe-initdb to fill the SQL and LDAP databases
with a defined set of test data. As opposed to the previous
tests/initdb.py script, it is callable from an installed instance of
fsfe-cd-back, so it can be used more easily in the staging environment.

* docker-compose is told to always recreate containers so we always
start with freshly initialized databases and LDAP servers, which we can
then fill with our test data.

.drone.yml

@@ -22,13 +22,13 @@ steps:
   - name: fsfe-cd-back-quality
     image: fsfe-cd-back-quality
     pull: never
+    environment:
+      DATABASE_URL: "sqlite://"
     commands:
       - cd back
       - isort --check-only --diff
       - pylama
-      - python3 -m tests.initdb
       - pytest
-      - rm test.db
     depends_on:
       - quality-images
 
@@ -81,7 +81,7 @@ steps:
       - name: dockersock
         path: /run/user/1001/docker.sock
     commands:
-      - docker-compose -p fsfe-cd --file docker-compose.staging.yml up --build --detach
+      - docker-compose -p fsfe-cd --file docker-compose.staging.yml up --build --force-recreate --detach
     depends_on:
       - quality-images
       - fsfe-cd-back-quality

Makefile

@@ -40,12 +40,12 @@ help:
 
 dev: ##@development Start all containers and show their logs.
 	@rm -f back/test.db
-	@USER_ID=$${USER_ID:-`id -u`} GROUP_ID=$${GROUP_ID:-`id -g`} docker-compose -f docker-compose.development.yml up --build
+	@USER_ID=$${USER_ID:-`id -u`} GROUP_ID=$${GROUP_ID:-`id -g`} docker-compose -f docker-compose.development.yml up --build --force-recreate
 .PHONY: dev
 
 dev.up: ##@development Start all containers and detach.
 	@rm -f back/test.db
-	@USER_ID=$${USER_ID:-`id -u`} GROUP_ID=$${GROUP_ID:-`id -g`} docker-compose -f docker-compose.development.yml up --build --detach
+	@USER_ID=$${USER_ID:-`id -u`} GROUP_ID=$${GROUP_ID:-`id -g`} docker-compose -f docker-compose.development.yml up --build --force-recreate --detach
 .PHONY: dev.up
 
 dev.down: ##@development Stop all containers.

back/.drone.yml

@@ -36,12 +36,12 @@ steps:
 - name: quality
   pull: if-not-exists
   image: fsfe-cd-back-quality
+  environment:
+    DATABASE_URL: sqlite://
   commands:
   - isort --check-only --diff
   - pylama
-  - python3 -m tests.initdb
   - pytest
-  - rm test.db
 
 - name: deploy
   pull: if-not-exists

back/Makefile

@@ -56,7 +56,7 @@ applyisort:  ##@development Apply a correct Python import sort inline.
 
 testdb:  ##@development Set up the test database.
 	@rm -f test.db
-	@pipenv run python3 -m tests.initdb
+	@PYTHONPATH="." PATH="bin:$$PATH" pipenv run fsfe-initdb
 .PHONY: testdb
 
 flask:  ##@development Run the Flask built-in web server.
@@ -79,8 +79,8 @@ lint:  ##@quality Check the Python source code for coding standards compliance.
 	@pipenv run pylama
 .PHONY: lint
 
-pytest: testdb  ##@quality Run the functional tests.
-	@pipenv run pytest --cov=$(SOURCE_DIR)
+pytest:  ##@quality Run the functional tests.
+	@DATABASE_URL="sqlite://" pipenv run pytest --cov=$(SOURCE_DIR)
 	@pipenv run coverage html
 .PHONY: pytest
 

back/bin/fsfe-initdb

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# =============================================================================
+# Initialize a database with test data for automatic and iteractive tests
+# =============================================================================
+# This file is part of the FSFE Community Database Backend.
+#
+# Copyright © 2017-2021 Free Software Foundation Europe <contact@fsfe.org>
+#
+# The FSFE Community Database Backend is free software: you can redistribute it
+# and/or modify it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the License,
+# or (at your option) any later version.
+#
+# The FSFE Community Database Backend is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+from fsfe_cd_back.data.initdb import initdb
+
+
+initdb()

back/doc/tools.md

@@ -361,6 +361,15 @@ Send another monthly statistics email.
 Runs in the morning of the first of each month.
 
 
+## Administration
+
+### fsfe-initdb
+
+Initialize the database and fill it with some test data. This is intended to be
+used for development and staging environments and will be called automatically
+in their setup.
+
+
 ## Selectors
 
 Some commands allow the use of selectors to define the affected people. If more

back/fsfe_cd_back/data/initdb.py

@@ -19,9 +19,6 @@
 from datetime import date
 from unittest.mock import patch
 
-from mockldap import MockLdap
-
-from fsfe_cd_back import config
 from fsfe_cd_back.data import Country, Person, session
 from fsfe_cd_back.data.basic import Base, Counter, engine
 
@@ -32,19 +29,6 @@ from fsfe_cd_back.data.basic import Base, Counter, engine
 
 @patch("smtplib.SMTP")
 def initdb(SMTP):
-    mock_ldap = MockLdap(
-            {
-                "dc=fsfe,dc=org": {
-                    "objectClass": ["top", "domain"],
-                    "dc": ["fsfe"]},
-                "cn=admin,dc=fsfe,dc=org": {
-                    "userPassword": [config.LDAP_PASSWORD],
-                    "cn": ["admin"]},
-                "ou=fellowship,dc=fsfe,dc=org": {
-                    "objectClass": ["organizationalUnit", "top"],
-                    "ou": ["fellowship"]}})
-    mock_ldap.start()
-
     Base.metadata.create_all(engine)
 
     session.add(Counter(name='payref'))
@@ -101,12 +85,3 @@ def initdb(SMTP):
     session.add(daniel)
 
     session.commit()
-
-    mock_ldap.stop()
-
-
-# =============================================================================
-# Main code
-# =============================================================================
-
-initdb()

back/tests/conftest.py

@@ -24,9 +24,38 @@ from requests import Response
 
 from fsfe_cd_back import config
 from fsfe_cd_back.data.basic import engine, session
+from fsfe_cd_back.data.initdb import initdb
 from fsfe_cd_back.server import create_app
 
 
+# =============================================================================
+# Fixture for creating and populating the test database
+# =============================================================================
+
+@pytest.fixture(scope="session", autouse=True)
+def testdb():
+    mock_ldap = MockLdap(
+            {
+                "dc=fsfe,dc=org": {
+                    "objectClass": ["top", "domain"],
+                    "dc": ["fsfe"]},
+                "cn=admin,dc=fsfe,dc=org": {
+                    "userPassword": [config.LDAP_PASSWORD],
+                    "cn": ["admin"]},
+                "ou=fellowship,dc=fsfe,dc=org": {
+                    "objectClass": ["organizationalUnit", "top"],
+                    "ou": ["fellowship"]}})
+    mock_ldap.start()
+
+    connection = engine.connect()
+    session(bind=connection)
+    initdb()
+    session.remove()
+    connection.close()
+
+    mock_ldap.stop()
+
+
 # =============================================================================
 # Fixtures for mocking connections to external services
 # =============================================================================

docker-compose.development.yml

@@ -105,9 +105,7 @@ services:
         group_id: "${GROUP_ID}"
     volumes:
       - "./back:/home/fsfe/src"
-    command: >
-      bash -c "python3 -m tests.initdb \
-      && sleep infinity"
+    command: sleep infinity
 
   # This is the main container
   fsfe-cd-back:
@@ -125,16 +123,15 @@ services:
       FLASK_RUN_PORT: 19000
       MAIL_SERVER: "fsfe-cd-smtp-mock"
       OIDCP_URL: "http://fsfe-cd-auth:19001/userinfo"
-      DATABASE_URL: "sqlite:////home/fsfe/src/test.db"
       LDAP_URL: "ldap://fsfe-cd-ldap-mock:1389"
       LDAP_PASSWORD: "admin"
     ports:
       - "19000:19000"
-    command: flask run
+    command: /bin/sh -c "src/bin/fsfe-initdb && flask run"
     volumes:
       - "./back:/home/fsfe/src"
     depends_on:
-      - fsfe-cd-back-quality
+      - fsfe-cd-ldap-mock
 
   # ===========================================================================
   # Authentication server (fsfe-cd-auth)

docker-compose.staging.yml

@@ -84,23 +84,10 @@ services:
   # Backend (fsfe-cd-back)
   # ===========================================================================
 
-  fsfe-cd-db-prep:
-    image: fsfe-cd-db-prep
-    container_name: fsfe-cd-db-prep
-    build:
-      context: back
-      target: base
-    environment:
-      DATABASE_URL: "postgresql://fsfe:${SECRET_KEY}@fsfe-cd-db/fsfe-staging"
-    command: bash -c "python3 -m tests.initdb"
-    depends_on:
-      - fsfe-cd-db
-
   fsfe-cd-back:
     image: fsfe-cd-back
     build:
       context: back
-      target: prod
     container_name: fsfe-cd-back
     environment:
       FLASK_ENV: "development"
@@ -111,7 +98,7 @@ services:
       LDAP_PASSWORD: "admin"
     ports:
       - "19000:19000"
-    command: gunicorn --bind 0.0.0.0:19000 'fsfe_cd_back.server:create_app()'
+    command: /bin/sh -c "fsfe-initdb && gunicorn --bind 0.0.0.0:19000 'fsfe_cd_back.server:create_app()'"
     volumes:
       - "/srv/fsfe-cd/back:/home/fsfe/data:rw"
     depends_on:

ldap-mock/custom/3_alice.ldif

@@ -1,8 +0,0 @@
-dn: uid=alice,ou=fellowship,dc=fsfe,dc=org
-objectClass: inetOrgPerson
-cn: alice
-sn: alice
-uid: alice
-userPassword: passw0rd
-email: alice@fsfe.org
-mail: alice@example.com