Added documentation with mkdocs

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+max_line_length = 80
+trim_trailing_whitespace = true

.github/pull_request_template.md

@@ -17,5 +17,6 @@ CCAP-XXX
 
 #### ✅ Completion tasks
 
+- [ ] Updated documentation
 - [ ] Added relevant tests
 - [ ] Meets acceptance criteria

.github/workflows/docs.yaml

@@ -0,0 +1,33 @@
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    name: Deploy Documentation
+    environment: 'dev'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Install plantuml
+        run: sudo apt-get install -y plantuml
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material pymdown-extensions plantuml_markdown
+      - run: mkdocs build
+      - run: aws s3 sync ./site "s3://${{ env.BUCKET_NAME || 'dev.docs.cfa.codes' }}/${{ env.PREFIX || 'document-transfer-service' }}"

.gitignore

@@ -4,6 +4,8 @@
 !sample.env
 coverage/
 log/
+out/
+site/
 
 # Ignore Byebug command history file.
 .byebug_history
@@ -21,4 +23,3 @@ log/
 .idea/dataSources.xml
 *.iml
 *.iws
-out/

.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
     - 'db/*schema.rb'
     - 'out/**/*'
     - 'vendor/**/*'
+    - 'doc/**/*'
 
 # Transient properties get mistaken for associations in FactoryBot.
 # See https://github.com/rubocop/rubocop-factory_bot/issues/73

Gemfile.lock

@@ -13,6 +13,7 @@ PATH
       activesupport (~> 7.1.0)
       adal (~> 1.0)
       bcrypt (~> 3.1)
+      configsl (~> 1.0)
       daemons (~> 1.4)
       delayed_job (~> 4.1)
       faraday (~> 2.9)
@@ -58,6 +59,8 @@ GEM
     bigdecimal (3.1.8)
     coderay (1.1.3)
     concurrent-ruby (1.3.3)
+    configsl (1.0.1)
+      facets (~> 3.1)
     connection_pool (2.4.1)
     csv (3.3.0)
     daemons (1.4.1)
@@ -83,6 +86,7 @@ GEM
       zeitwerk (~> 2.6)
     et-orbi (1.2.11)
       tzinfo
+    facets (3.1.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     faker (3.4.1)

README.md

@@ -38,7 +38,7 @@ service.
 Using docker compose, the application code will be mounted from your local
 system to the running container. This will allow you to make changes to the
 code and see them reflected in the running service without having to rebuild the
-image. Using docker compose will launch the api, the worker, and a database. 
+image. Using docker compose will launch the api, the worker, and a database.
 
 To run the service with docker compose, make sure you have [Docker Desktop]
 installed and run the following:
@@ -132,16 +132,34 @@ bundle exec rake db:drop
 See the [API documentation][api] for information on how to interact with the
 service.
 
-[.env]: ./sample.env
-[api]: ./doc/api.md
-[create-key]: ./doc/runbooks/create_auth_key.md
-[database]: ./doc/database.md
-[destination]: ./doc/destinations.md
-[Dockerfile]: ./Dockerfile
-[docker compose]: ./docker-compose.yaml
+## Documentation
+
+Necessary documentation to operate, use, maintain, and contribute to the service
+is included in this repository. The majority of these documents are written in
+Markdown and can be rendered directly in GitHub or you favorite IDE. However,
+the documentation as a whole is meant to be converted to a static site using
+[MkDocs].
+
+In order to view the documentation in its intended form locally, you can use the
+included docker container. Simply run the following:
+
+```bash
+docker compose --profile docs up -d
+```
+
+The documentation should then be available at http://localhost:8000.
+
+[.env]: https://github.com/codeforamerica/document-transfer-service/blob/main/sample.env
+[api]: ./api.md
+[create-key]: ./runbooks/create_auth_key.md
+[database]: ./database.md
+[destination]: ./api/destinations.md
+[Dockerfile]: https://github.com/codeforamerica/document-transfer-service/blob/main/Dockerfile
+[docker compose]: https://github.com/codeforamerica/document-transfer-service/blob/main/docker-compose.yaml
 [Docker Desktop]: https://docs.docker.com/desktop/
+[mkdocs]: https://www.mkdocs.org/
 [omz]: https://ohmyz.sh/
-[ruby-version]: ./.ruby-version
+[ruby-version]: https://github.com/codeforamerica/document-transfer-service/blob/main/.ruby-version
 [rvm]: https://rvm.io/
-[source]: ./doc/sources.md
-[worker]: ./doc/worker.md
+[source]: ./api/sources.md
+[worker]: ./worker.md

doc/api.md

@@ -73,7 +73,7 @@ middleware for rack. The logger utilizes [semantic logging][semantic_logger] in
 JSON format to provide easily parsable log entries.
 
 [authentication]: ./api/authentication.md
-[destination]: ./destinations.md
+[destination]: ./api/destinations.md
 [semantic_logger]: https://logger.rocketjob.io/
-[source]: ./sources.md
-[spec]: ../openapi.yaml
+[source]: ./api/sources.md
+[spec]: https://github.com/codeforamerica/document-transfer-service/blob/main/openapi.yaml

doc/architecture.md

@@ -0,0 +1,180 @@
+# Architecture
+
+The Document Transfer Service is designed to be as lightweight as possible. As
+such, we have chosen to use [grape] as the API framework. Grape is simple
+microservice framework for ruby. It's opinionated, but less so than Rails, with
+minimal dependencies.
+
+## System Context
+
+Although the exposed APIs are meant to be simple and generic, and could
+therefore be used by any system, the primary use case is to support the
+application process for benefits within America's [safety net]. As such, we'll
+use this as our context for system documentation.
+
+Within this larger system, the Document Transfer Service acts as a component
+dedicated to the secure transfer of documents between the applicant and the
+benefits system. This could be a state run system, or a third party system
+authorized to process benefits applications.
+
+```plantuml
+!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml
+
+title Document Transfer Service - System Context
+AddRelTag("optional", $textColor="gray", $lineColor="gray", $lineStyle = DashedLine())
+
+Person(applicant, "Applicant")
+System_Ext(benefits_app, "Benefits Application", "Digital application for benefits")
+System(doc_transfer, "Document Transfer Service", "A microservice to securely transfer documents")
+System_Ext(benefits_system, "Benefits System", "System that processes benefits applications")
+
+Rel_R(applicant, benefits_app, "Applies")
+Rel_R(benefits_app, doc_transfer, "Requests document transfer")
+Rel_D(doc_transfer, benefits_system, "Sends")
+Rel(benefits_app, benefits_system, "Submits", $tags="optional")
+Rel(benefits_system, applicant, "Notifies", $tags="optional")
+
+footer Last updated 2024-09-04 for Document Transfer Service v1.0.0
+```
+
+The benefits application is typically a web form application that simplifies the
+benefits application process for the application. The application will upload
+their required documents to this system, which will then make an API call to
+the Document Transfer Service to transfer the documents to the appropriate
+destination. Typically, this call will be done asynchronously, so the applicant
+is not waiting for the transfer to complete.
+
+## The application
+
+The application can be broken down into three [containers][c4-containers]:
+
+- The [**API**][api] that handles the requests
+- The [**worker**][worker] that processes background jobs
+- The **database** that stores the transfer requests
+
+```plantuml
+!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml
+
+title Document Transfer Service - Container Diagram
+AddRelTag("optional", $textColor="gray", $lineColor="gray", $lineStyle = DashedLine())
+
+Person(applicant, "Applicant")
+System_Ext(benefits_app, "Benefits Application", "Digital application for benefits")
+System_Ext(benefits_system, "Benefits System", "System that processes benefits applications")
+
+System_Boundary(doc_transfer, "Document Transfer Service") {
+  Container(api, "API", "Handles incoming requests")
+  ContainerQueue(worker, "Worker", "Processes document transfers")
+  ContainerDb(postgres, "PostgreSQL", "Stores transfer requests")
+
+  Rel(api, postgres, "Record and queue transfer requests")
+  Rel(worker, postgres, "Process background jobs")
+}
+
+Rel_R(applicant, benefits_app, "Applies")
+Rel_D(benefits_app, api, "Requests document transfer")
+Rel_L(worker, benefits_system, "Sends")
+Rel(benefits_app, benefits_system, "Submits", $tags="optional")
+Rel_U(benefits_system, applicant, "Notifies", $tags="optional")
+
+footer Last updated 2024-09-04 for Document Transfer Service v1.0.0
+```
+
+### API
+
+The API is the entry point for the service. It handles incoming requests,
+records them in the database, and queues the transfer job for the worker to
+process.
+
+See the [API Architecture][api] for more details.
+
+### Worker
+
+The worker is responsible for processing the background jobs. It reads the
+job queue from the database and processes jobs as they are available. Transfer
+jobs are responsible for retrieving the source document and transferring it to
+the destination.
+
+See the [Worker Architecture][worker] for more details.
+
+### Database
+
+!!! warning "Sensitive Data Warning"
+
+    While the service does not intend to store any PII (Personally
+    Identifiable Information) or other sensitive data, any data passed in the source
+    and destination parameters will be stored until the transfer has completed.
+
+The Document Transfer service is designed to work with a [PostgreSQL] database.
+Other database backends may work, but only the PostgreSQL drivers are included.
+
+The database is used to store consumer details -- such as name and
+authentication keys -- and the transfer requests. The service _does not_ store
+or otherwise handle the contents of the documents being transferred.
+
+Request data is stored in two pieces: the request itself and the background job
+that performs the transfer. This allows the service to track the status of
+transfers, while retaining the source and destination information for a minimum
+length of time as jobs are deleted once they have been completed.
+
+See the [database] documentation for more details on how the database is
+configured.
+
+```mermaid
+sequenceDiagram
+    critical Request a document transfer
+        activate Consumer
+        activate API
+        activate Database
+        Consumer->>API: Request transfer
+        API->>Database: Record transfer
+        API->>Database: Enqueue background job
+        API->>Consumer: Return transfer id
+        deactivate Consumer
+    end
+
+    critical Background job processing
+        activate Worker
+        loop until success or max attempts
+            Worker->>Database: Reserve job
+            activate Source
+            Worker->>Source: Fetch document
+            deactivate Source
+            activate Destination
+            Worker->>Destination: Transfer document
+            deactivate Destination
+            alt Transfer succeeded
+                Worker->>Database: Update request status
+                Worker->>Database: Mark job complete
+            else Transfer failed
+                alt Max attempts reached
+                    rect rgb(191, 223, 255)
+                    Worker->>Database: Update request status
+                    Worker->>Database: Mark job failed
+                    end
+                else Retry transfer
+                    Worker->>Database: Mark job failed
+                end
+            end
+        end
+        deactivate Worker
+    end
+
+    opt Check transfer status
+        activate Consumer
+        Consumer->>API: Check transfer status
+        API->>Database: Retrieve transfer status
+        API->>Consumer: Return transfer status
+        deactivate Database
+        deactivate API
+        deactivate Consumer
+    end
+```
+
+[api]: architecture/api.md
+[c4-containers]: https://c4model.com/#ContainerDiagram
+[database]: database.md
+[grape]: https://github.com/ruby-grape/grape
+[Postgresql]: https://www.postgresql.org/
+[safety net]: https://codeforamerica.org/programs/social-safety-net/
+[worker]: architecture/worker.md