Use poetry instead of requirements files (#4151)

Author: mbertrand

.github/workflows/ci.yml

@@ -49,22 +49,26 @@ jobs:
       - name: Apt install
         run: cat Aptfile | sudo xargs apt-get install
 
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
       - uses: actions/setup-python@v4
         with:
           python-version: '3.11.4'
-          cache: 'pip'
-          cache-dependency-path: |
-            **/requirements.txt
-            **/test_requirements.txt
+          cache: 'poetry'
 
       - name: Install dependencies
-        run: pip install -r requirements.txt -r test_requirements.txt
+        run: poetry install --no-interaction
 
       - name: Code formatting
-        run: black --check .
+        run: poetry run black --check .
 
       - name: Lint
-        run: pylint ./**/*.py
+        run: poetry run pylint ./**/*.py
 
       - name: Create test local state
         run: ./scripts/test/stub-data.sh

Dockerfile

@@ -18,10 +18,21 @@ RUN mkdir /src
 RUN adduser --disabled-password --gecos "" mitodl
 RUN mkdir /var/media && chown -R mitodl:mitodl /var/media
 
+# Poetry env configuration
+ENV  \
+  # poetry:
+  POETRY_VERSION=1.5.1 \
+  POETRY_VIRTUALENVS_CREATE=false \
+  POETRY_CACHE_DIR='/tmp/cache/poetry'
+
+# Install poetry
+RUN pip install "poetry==$POETRY_VERSION"
+
 # Install project packages
-COPY requirements.txt /tmp/requirements.txt
-COPY test_requirements.txt /tmp/test_requirements.txt
-RUN pip install -r requirements.txt -r test_requirements.txt
+COPY pyproject.toml /src
+COPY poetry.lock /src
+WORKDIR /src
+RUN poetry install
 
 # Add project
 COPY . /src
@@ -31,9 +42,6 @@ RUN chown -R mitodl:mitodl /src
 RUN apt-get clean && apt-get purge
 USER mitodl
 
-# Set pip cache folder, as it is breaking pip when it is on a shared volume
-ENV XDG_CACHE_HOME /tmp/.cache
-
 EXPOSE 8063
 ENV PORT 8063
 CMD uwsgi uwsgi.ini

app.json

@@ -19,6 +19,9 @@
     },
     {
       "url": "https://github.com/heroku/heroku-buildpack-nginx"
+    },
+    {
+      "url": "https://github.com/moneymeets/python-poetry-buildpack"
     }
   ],
   "description": "open-discussions",

channels/views/posts_test.py

@@ -9,7 +9,7 @@
 from rest_framework.request import Request
 
 from profiles.utils import image_uri
-from channels.factories.models import LinkMetaFactory
+from channels.factories.models import LinkMetaFactory, PostFactory
 from channels.constants import (
     EXTENDED_POST_TYPE_ARTICLE,
     VALID_POST_SORT_TYPES,
@@ -437,6 +437,7 @@ def test_get_post_no_profile(
 def test_get_post_forbidden(client, logged_in_profile):
     """Get a post the user doesn't have permission to"""
     post_id = "adc"
+    PostFactory.create(post_id=post_id)
     url = reverse("post-detail", kwargs={"post_id": post_id})
     resp = client.get(url)
     assert resp.status_code == status.HTTP_403_FORBIDDEN
@@ -445,6 +446,7 @@ def test_get_post_forbidden(client, logged_in_profile):
 def test_get_post_not_found(client, logged_in_profile):
     """Get a post the user doesn't have permission to"""
     post_id = "missing"
+    PostFactory.create(post_id=post_id)
     url = reverse("post-detail", kwargs={"post_id": post_id})
     resp = client.get(url)
     assert resp.status_code == status.HTTP_404_NOT_FOUND

course_catalog/etl/utils.py

@@ -19,7 +19,6 @@
 import pytz
 import rapidjson
 import requests
-from boto.s3.bucket import Bucket
 from django.conf import settings
 from django.utils.functional import SimpleLazyObject
 from tika import parser as tika_parser
@@ -442,7 +441,7 @@ def get_learning_course_bucket_name(platform: str) -> str:
     return bucket_names.get(platform)
 
 
-def get_learning_course_bucket(platform: str) -> Bucket:
+def get_learning_course_bucket(platform: str) -> object:
     """
     Get the platform's learning course S3 Bucket holding content file data
 

course_catalog/etl/youtube_test.py

@@ -571,7 +571,7 @@ def test_get_youtube_transcripts_with_multiple_consecutive_failures(mocker):
     mock_caption_call = mocker.patch(
         "course_catalog.etl.youtube.get_captions_for_video"
     )
-    mock_caption_call.side_effect = pytube.exceptions.VideoUnavailable
+    mock_caption_call.side_effect = pytube.exceptions.VideoUnavailable(1)
 
     mock_parse_call = mocker.patch("course_catalog.etl.youtube.parse_video_captions")
     mock_upsert_video = mocker.patch("course_catalog.etl.youtube.upsert_video")

course_catalog/views_stafflist_test.py

@@ -15,7 +15,7 @@
 from course_catalog.utils import update_editor_group
 from open_discussions.factories import UserFactory
 
-# pylint:disable=redefined-outer-name
+# pylint:disable=redefined-outer-name,unused-argument
 
 
 @pytest.fixture()
@@ -118,7 +118,9 @@ def test_staff_list_endpoint_create(  # pylint: disable=too-many-arguments
 @pytest.mark.parametrize("is_public", [True, False])
 @pytest.mark.parametrize("is_editor", [True, False])
 @pytest.mark.parametrize("update_topics", [True, False])
-def test_staff_list_endpoint_patch(client, update_topics, is_public, is_editor):
+def test_staff_list_endpoint_patch(
+    client, mock_staff_list_index, update_topics, is_public, is_editor
+):
     """Test stafflist endpoint for updating a StaffList"""
     [original_topic, new_topic] = CourseTopicFactory.create_batch(2)
     user = UserFactory.create()