diff --git a/.gitignore b/.gitignore index 6f329c7393..61351715a1 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,6 @@ scripts/ # i18n auto-generated types (regenerated on every build) packages/i18n/src/types/keys.generated.ts + +# Local security advisory notes (not for version control) +/advisories.md diff --git a/apps/api/plane/app/serializers/api.py b/apps/api/plane/app/serializers/api.py index 05c6198f59..71ffd442bf 100644 --- a/apps/api/plane/app/serializers/api.py +++ b/apps/api/plane/app/serializers/api.py @@ -22,6 +22,7 @@ class APITokenSerializer(BaseSerializer): "is_active", "last_used", "user_type", + "allowed_rate_limit", ] diff --git a/apps/api/plane/bgtasks/cleanup_task.py b/apps/api/plane/bgtasks/cleanup_task.py index 407a67ca69..3414438ccf 100644 --- a/apps/api/plane/bgtasks/cleanup_task.py +++ b/apps/api/plane/bgtasks/cleanup_task.py @@ -5,19 +5,16 @@ # Python imports from datetime import timedelta import logging -from typing import List, Dict, Any, Callable, Optional -import os +from typing import Callable, Iterable # Django imports +from django.conf import settings from django.utils import timezone from django.db.models import F, Window, Subquery from django.db.models.functions import RowNumber # Third party imports from celery import shared_task -from pymongo.errors import BulkWriteError -from pymongo.collection import Collection -from pymongo.operations import InsertOne # Module imports from plane.db.models import ( @@ -27,7 +24,6 @@ from plane.db.models import ( IssueDescriptionVersion, WebhookLog, ) -from plane.settings.mongo import MongoConnection from plane.utils.exception_logger import log_exception @@ -35,285 +31,75 @@ logger = logging.getLogger("plane.worker") BATCH_SIZE = 500 -def get_mongo_collection(collection_name: str) -> Optional[Collection]: - """Get MongoDB collection if available, otherwise return None.""" - if not MongoConnection.is_configured(): - logger.info("MongoDB not configured") - return None - - try: - mongo_collection = MongoConnection.get_collection(collection_name) - logger.info(f"MongoDB collection '{collection_name}' connected successfully") - return mongo_collection - except Exception as e: - logger.error(f"Failed to get MongoDB collection: {str(e)}") - log_exception(e) - return None - - -def flush_to_mongo_and_delete( - mongo_collection: Optional[Collection], - buffer: List[Dict[str, Any]], - ids_to_delete: List[int], - model, - mongo_available: bool, -) -> None: - """ - Inserts a batch of records into MongoDB and deletes the corresponding rows from PostgreSQL. - """ - if not buffer: - logger.debug("No records to flush - buffer is empty") - return - - logger.info(f"Starting batch flush: {len(buffer)} records, {len(ids_to_delete)} IDs to delete") - - mongo_archival_failed = False - - # Try to insert into MongoDB if available - if mongo_collection is not None and mongo_available: - try: - mongo_collection.bulk_write([InsertOne(doc) for doc in buffer]) - except BulkWriteError as bwe: - logger.error(f"MongoDB bulk write error: {str(bwe)}") - log_exception(bwe) - mongo_archival_failed = True - - # If MongoDB is available and archival failed, log the error and return - if mongo_available and mongo_archival_failed: - logger.error(f"MongoDB archival failed for {len(buffer)} records") - return - - # Delete from PostgreSQL - delete() returns (count, {model: count}) - delete_result = model.all_objects.filter(id__in=ids_to_delete).delete() - deleted_count = delete_result[0] if delete_result and isinstance(delete_result, tuple) else 0 - logger.info(f"Batch flush completed: {deleted_count} records deleted") - - def process_cleanup_task( - queryset_func: Callable, - transform_func: Callable[[Dict], Dict], + queryset_func: Callable[[], Iterable], model, task_name: str, - collection_name: str, ): """ - Generic function to process cleanup tasks. + Batch-delete expired rows for the given model from PostgreSQL. Args: - queryset_func: Function that returns the queryset to process - transform_func: Function to transform each record for MongoDB - model: Django model class - task_name: Name of the task for logging - collection_name: MongoDB collection name + queryset_func: Callable returning an iterable of primary keys to delete. + model: Django model class. + task_name: Name of the task for logging. """ logger.info(f"Starting {task_name} cleanup task") - # Get MongoDB collection - mongo_collection = get_mongo_collection(collection_name) - mongo_available = mongo_collection is not None - - # Get queryset - queryset = queryset_func() - - # Process records in batches - buffer: List[Dict[str, Any]] = [] - ids_to_delete: List[int] = [] - total_processed = 0 + total_deleted = 0 total_batches = 0 + batch: list = [] - for record in queryset: - # Transform record for MongoDB - buffer.append(transform_func(record)) - ids_to_delete.append(record["id"]) - - # Flush batch when it reaches BATCH_SIZE - if len(buffer) >= BATCH_SIZE: - total_batches += 1 - flush_to_mongo_and_delete( - mongo_collection=mongo_collection, - buffer=buffer, - ids_to_delete=ids_to_delete, - model=model, - mongo_available=mongo_available, - ) - total_processed += len(buffer) - buffer.clear() - ids_to_delete.clear() - - # Process final batch if any records remain - if buffer: + def flush(ids: list) -> None: + nonlocal total_deleted, total_batches + if not ids: + return total_batches += 1 - flush_to_mongo_and_delete( - mongo_collection=mongo_collection, - buffer=buffer, - ids_to_delete=ids_to_delete, - model=model, - mongo_available=mongo_available, - ) - total_processed += len(buffer) + try: + # `all_objects` is a plain manager, so this is a hard delete — rows + # are removed from PostgreSQL immediately rather than soft-deleted. + delete_result = model.all_objects.filter(id__in=ids).delete() + deleted = delete_result[0] if isinstance(delete_result, tuple) else 0 + total_deleted += deleted + except Exception as e: + # Log and skip a failed batch rather than aborting the whole run, so + # a single bad batch doesn't block cleanup of the remaining rows. + log_exception(e) + + for record_id in queryset_func(): + batch.append(record_id) + if len(batch) >= BATCH_SIZE: + flush(batch) + batch = [] + + # Flush the final partial batch + flush(batch) logger.info( f"{task_name} cleanup task completed", - extra={ - "total_records_processed": total_processed, - "total_batches": total_batches, - "mongo_available": mongo_available, - "collection_name": collection_name, - }, + extra={"total_records_deleted": total_deleted, "total_batches": total_batches}, ) -# Transform functions for each model -def transform_api_log(record: Dict) -> Dict: - """Transform API activity log record.""" - return { - "id": str(record["id"]), - "created_at": str(record["created_at"]) if record.get("created_at") else None, - "token_identifier": str(record["token_identifier"]), - "path": record["path"], - "method": record["method"], - "query_params": record.get("query_params"), - "headers": record.get("headers"), - "body": record.get("body"), - "response_code": record["response_code"], - "response_body": record["response_body"], - "ip_address": record["ip_address"], - "user_agent": record["user_agent"], - "created_by_id": str(record["created_by_id"]), - } - - -def transform_email_log(record: Dict) -> Dict: - """Transform email notification log record.""" - return { - "id": str(record["id"]), - "created_at": str(record["created_at"]) if record.get("created_at") else None, - "receiver_id": str(record["receiver_id"]), - "triggered_by_id": str(record["triggered_by_id"]), - "entity_identifier": str(record["entity_identifier"]), - "entity_name": record["entity_name"], - "data": record["data"], - "processed_at": (str(record["processed_at"]) if record.get("processed_at") else None), - "sent_at": str(record["sent_at"]) if record.get("sent_at") else None, - "entity": record["entity"], - "old_value": str(record["old_value"]), - "new_value": str(record["new_value"]), - "created_by_id": str(record["created_by_id"]), - } - - -def transform_page_version(record: Dict) -> Dict: - """Transform page version record.""" - return { - "id": str(record["id"]), - "created_at": str(record["created_at"]) if record.get("created_at") else None, - "page_id": str(record["page_id"]), - "workspace_id": str(record["workspace_id"]), - "owned_by_id": str(record["owned_by_id"]), - "description_html": record["description_html"], - "description_binary": record["description_binary"], - "description_stripped": record["description_stripped"], - "description_json": record["description_json"], - "sub_pages_data": record["sub_pages_data"], - "created_by_id": str(record["created_by_id"]), - "updated_by_id": str(record["updated_by_id"]), - "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None, - "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None), - } - - -def transform_issue_description_version(record: Dict) -> Dict: - """Transform issue description version record.""" - return { - "id": str(record["id"]), - "created_at": str(record["created_at"]) if record.get("created_at") else None, - "issue_id": str(record["issue_id"]), - "workspace_id": str(record["workspace_id"]), - "project_id": str(record["project_id"]), - "created_by_id": str(record["created_by_id"]), - "updated_by_id": str(record["updated_by_id"]), - "owned_by_id": str(record["owned_by_id"]), - "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None), - "description_binary": record["description_binary"], - "description_html": record["description_html"], - "description_stripped": record["description_stripped"], - "description_json": record["description_json"], - "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None, - } - - -def transform_webhook_log(record: Dict): - """Transfer webhook logs to a new destination.""" - return { - "id": str(record["id"]), - "created_at": str(record["created_at"]) if record.get("created_at") else None, - "workspace_id": str(record["workspace_id"]), - "webhook": str(record["webhook"]), - # Request - "event_type": str(record["event_type"]), - "request_method": str(record["request_method"]), - "request_headers": str(record["request_headers"]), - "request_body": str(record["request_body"]), - # Response - "response_status": str(record["response_status"]), - "response_body": str(record["response_body"]), - "response_headers": str(record["response_headers"]), - # retry count - "retry_count": str(record["retry_count"]), - } - - -# Queryset functions for each cleanup task +# Queryset functions for each cleanup task — each yields primary keys to delete def get_api_logs_queryset(): - """Get API logs older than cutoff days.""" - cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30)) - cutoff_time = timezone.now() - timedelta(days=cutoff_days) + """Get API activity logs older than the API retention window.""" + cutoff_time = timezone.now() - timedelta(days=settings.API_ACTIVITY_LOG_RETENTION_DAYS) logger.info(f"API logs cutoff time: {cutoff_time}") - return ( APIActivityLog.all_objects.filter(created_at__lte=cutoff_time) - .values( - "id", - "created_at", - "token_identifier", - "path", - "method", - "query_params", - "headers", - "body", - "response_code", - "response_body", - "ip_address", - "user_agent", - "created_by_id", - ) + .values_list("id", flat=True) .iterator(chunk_size=BATCH_SIZE) ) def get_email_logs_queryset(): - """Get email logs older than cutoff days.""" - cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30)) - cutoff_time = timezone.now() - timedelta(days=cutoff_days) + """Get email logs older than the email retention window.""" + cutoff_time = timezone.now() - timedelta(days=settings.EMAIL_LOG_RETENTION_DAYS) logger.info(f"Email logs cutoff time: {cutoff_time}") - return ( EmailNotificationLog.all_objects.filter(sent_at__lte=cutoff_time) - .values( - "id", - "created_at", - "receiver_id", - "triggered_by_id", - "entity_identifier", - "entity_name", - "data", - "processed_at", - "sent_at", - "entity", - "old_value", - "new_value", - "created_by_id", - ) + .values_list("id", flat=True) .iterator(chunk_size=BATCH_SIZE) ) @@ -334,22 +120,7 @@ def get_page_versions_queryset(): return ( PageVersion.all_objects.filter(id__in=Subquery(subq)) - .values( - "id", - "created_at", - "page_id", - "workspace_id", - "owned_by_id", - "description_html", - "description_binary", - "description_stripped", - "description_json", - "sub_pages_data", - "created_by_id", - "updated_by_id", - "deleted_at", - "last_saved_at", - ) + .values_list("id", flat=True) .iterator(chunk_size=BATCH_SIZE) ) @@ -370,52 +141,20 @@ def get_issue_description_versions_queryset(): return ( IssueDescriptionVersion.all_objects.filter(id__in=Subquery(subq)) - .values( - "id", - "created_at", - "issue_id", - "workspace_id", - "project_id", - "created_by_id", - "updated_by_id", - "owned_by_id", - "last_saved_at", - "description_binary", - "description_html", - "description_stripped", - "description_json", - "deleted_at", - ) + .values_list("id", flat=True) .iterator(chunk_size=BATCH_SIZE) ) def get_webhook_logs_queryset(): - """Get email logs older than cutoff days.""" - cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30)) - cutoff_time = timezone.now() - timedelta(days=cutoff_days) + """Get webhook logs older than the webhook retention window.""" + cutoff_time = timezone.now() - timedelta(days=settings.WEBHOOK_LOG_RETENTION_DAYS) logger.info(f"Webhook logs cutoff time: {cutoff_time}") - return ( WebhookLog.all_objects.filter(created_at__lte=cutoff_time) - .values( - "id", - "created_at", - "workspace_id", - "webhook", - "event_type", - # Request - "request_method", - "request_headers", - "request_body", - # Response - "response_status", - "response_body", - "response_headers", - "retry_count", - ) .order_by("created_at") - .iterator(chunk_size=100) + .values_list("id", flat=True) + .iterator(chunk_size=BATCH_SIZE) ) @@ -424,10 +163,8 @@ def delete_api_logs(): """Delete old API activity logs.""" process_cleanup_task( queryset_func=get_api_logs_queryset, - transform_func=transform_api_log, model=APIActivityLog, task_name="API Activity Log", - collection_name="api_activity_logs", ) @@ -436,10 +173,8 @@ def delete_email_notification_logs(): """Delete old email notification logs.""" process_cleanup_task( queryset_func=get_email_logs_queryset, - transform_func=transform_email_log, model=EmailNotificationLog, task_name="Email Notification Log", - collection_name="email_notification_logs", ) @@ -448,10 +183,8 @@ def delete_page_versions(): """Delete excess page versions.""" process_cleanup_task( queryset_func=get_page_versions_queryset, - transform_func=transform_page_version, model=PageVersion, task_name="Page Version", - collection_name="page_versions", ) @@ -460,20 +193,16 @@ def delete_issue_description_versions(): """Delete excess issue description versions.""" process_cleanup_task( queryset_func=get_issue_description_versions_queryset, - transform_func=transform_issue_description_version, model=IssueDescriptionVersion, task_name="Issue Description Version", - collection_name="issue_description_versions", ) @shared_task def delete_webhook_logs(): - """Delete old webhook logs""" + """Delete old webhook logs.""" process_cleanup_task( queryset_func=get_webhook_logs_queryset, - transform_func=transform_webhook_log, model=WebhookLog, task_name="Webhook Log", - collection_name="webhook_logs", ) diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.py index 4a74e54bc5..3285e544a3 100644 --- a/apps/api/plane/bgtasks/logger_task.py +++ b/apps/api/plane/bgtasks/logger_task.py @@ -4,14 +4,12 @@ # Python imports import logging -from typing import Optional, Dict, Any +from typing import Dict, Any # Third party imports -from pymongo.collection import Collection from celery import shared_task # Django imports -from plane.settings.mongo import MongoConnection from plane.utils.exception_logger import log_exception from plane.db.models import APIActivityLog @@ -19,66 +17,9 @@ from plane.db.models import APIActivityLog logger = logging.getLogger("plane.worker") -def get_mongo_collection() -> Optional[Collection]: - """ - Returns the MongoDB collection for external API activity logs. - """ - if not MongoConnection.is_configured(): - logger.info("MongoDB not configured") - return None - - try: - return MongoConnection.get_collection("api_activity_logs") - except Exception as e: - logger.error(f"Error getting MongoDB collection: {str(e)}") - log_exception(e) - return None - - -def safe_decode_body(content: bytes) -> Optional[str]: - """ - Safely decodes request/response body content, handling binary data. - Returns "[Binary Content]" if the content is binary, or a string representation of the content. - Returns None if the content is None or empty. - """ - # If the content is None, return None - if content is None: - return None - - # If the content is an empty bytes object, return None - if content == b"": - return None - - # Check if content is binary by looking for common binary file signatures - if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"): - return "[Binary Content]" - - try: - return content.decode("utf-8") - except UnicodeDecodeError: - return "[Could not decode content]" - - -def log_to_mongo(log_document: Dict[str, Any]) -> bool: - """ - Logs the request to MongoDB if available. - """ - mongo_collection = get_mongo_collection() - if mongo_collection is None: - logger.error("MongoDB not configured") - return False - - try: - mongo_collection.insert_one(log_document) - return True - except Exception as e: - log_exception(e) - return False - - def log_to_postgres(log_data: Dict[str, Any]) -> bool: """ - Fallback to logging to PostgreSQL if MongoDB is unavailable. + Persist an external API request log to PostgreSQL. """ try: APIActivityLog.objects.create(**log_data) @@ -89,12 +30,12 @@ def log_to_postgres(log_data: Dict[str, Any]) -> bool: @shared_task -def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]) -> None: - """ - Process logs to save to MongoDB or Postgres based on the configuration +def process_logs(log_data: Dict[str, Any], **_: Any) -> None: """ + Persist external API request logs to PostgreSQL. - if MongoConnection.is_configured(): - log_to_mongo(mongo_log) - else: - log_to_postgres(log_data) + The catch-all kwargs keep this task signature compatible with jobs enqueued + by an older release (which passed a `mongo_log` argument), so in-flight tasks + don't fail during a rolling deploy. It can be dropped once no such jobs remain. + """ + log_to_postgres(log_data) diff --git a/apps/api/plane/bgtasks/webhook_task.py b/apps/api/plane/bgtasks/webhook_task.py index ddb5299fb5..02e6915d4e 100644 --- a/apps/api/plane/bgtasks/webhook_task.py +++ b/apps/api/plane/bgtasks/webhook_task.py @@ -53,7 +53,6 @@ from plane.license.utils.instance_value import get_email_configuration from plane.utils.email import generate_plain_text_from_html from plane.utils.exception_logger import log_exception from plane.utils.ip_address import validate_url -from plane.settings.mongo import MongoConnection SERIALIZER_MAPPER = { @@ -102,9 +101,6 @@ def save_webhook_log( retry_count: int, event_type: str, ) -> None: - # webhook_logs - mongo_collection = MongoConnection.get_collection("webhook_logs") - log_data = { "workspace_id": str(webhook.workspace_id), "webhook": str(webhook.id), @@ -118,27 +114,12 @@ def save_webhook_log( "retry_count": retry_count, } - mongo_save_success = False - if mongo_collection is not None: - try: - # insert the log data into the mongo collection - mongo_collection.insert_one(log_data) - logger.info("Webhook log saved successfully to mongo") - mongo_save_success = True - except Exception as e: - log_exception(e, warning=True) - logger.error(f"Failed to save webhook log: {e}") - mongo_save_success = False - - # if the mongo save is not successful, save the log data into the database - if not mongo_save_success: - try: - # insert the log data into the database - WebhookLog.objects.create(**log_data) - logger.info("Webhook log saved successfully to database") - except Exception as e: - log_exception(e, warning=True) - logger.error(f"Failed to save webhook log: {e}") + try: + WebhookLog.objects.create(**log_data) + logger.info("Webhook log saved successfully to database") + except Exception as e: + log_exception(e, warning=True) + logger.error(f"Failed to save webhook log: {e}") def get_model_data(event: str, event_id: Union[str, List[str]], many: bool = False) -> Dict[str, Any]: diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py index b8cf6f9c04..343409c4e1 100644 --- a/apps/api/plane/middleware/logger.py +++ b/apps/api/plane/middleware/logger.py @@ -3,12 +3,14 @@ # See the LICENSE file for details. # Python imports +import hashlib +import hmac import logging import time # Django imports +from django.conf import settings from django.http import HttpRequest -from django.utils import timezone # Third party imports from rest_framework.request import Request @@ -77,7 +79,7 @@ class RequestLoggerMiddleware: class APITokenLogMiddleware: """ - Middleware to log External API requests to MongoDB or PostgreSQL. + Middleware to log External API requests to PostgreSQL. """ def __init__(self, get_response): @@ -111,6 +113,20 @@ class APITokenLogMiddleware: except UnicodeDecodeError: return "[Could not decode content]" + # Headers whose values must never be persisted in plaintext logs + SENSITIVE_HEADERS = frozenset({"x-api-key", "authorization", "cookie"}) + + def _redacted_headers(self, request): + """ + Returns the request headers as a string with sensitive values redacted, + so that credentials such as the API key are never stored in plaintext. + """ + redacted = { + key: ("[REDACTED]" if key.lower() in self.SENSITIVE_HEADERS else value) + for key, value in request.headers.items() + } + return str(redacted) + def process_request(self, request, response, request_body): api_key_header = "X-Api-Key" api_key = request.headers.get(api_key_header) @@ -121,32 +137,25 @@ class APITokenLogMiddleware: try: log_data = { - "token_identifier": api_key, + # Tokenize the (high-entropy) API key into a stable, non-reversible + # identifier so logs can be correlated to a token without ever + # persisting the raw key. A keyed HMAC is used rather than a bare + # hash so the digest cannot be precomputed from a known key value. + "token_identifier": hmac.new( + settings.SECRET_KEY.encode(), api_key.encode(), hashlib.sha256 + ).hexdigest(), "path": request.path, "method": request.method, "query_params": request.META.get("QUERY_STRING", ""), - "headers": str(request.headers), + "headers": self._redacted_headers(request), "body": self._safe_decode_body(request_body) if request_body else None, "response_body": self._safe_decode_body(response.content) if response.content else None, "response_code": response.status_code, "ip_address": get_client_ip(request=request), "user_agent": request.META.get("HTTP_USER_AGENT", None), } - user_id = ( - str(request.user.id) - if getattr(request, "user") and getattr(request.user, "is_authenticated", False) - else None - ) - # Additional fields for MongoDB - mongo_log = { - **log_data, - "created_at": timezone.now(), - "updated_at": timezone.now(), - "created_by": user_id, - "updated_by": user_id, - } - process_logs.delay(log_data=log_data, mongo_log=mongo_log) + process_logs.delay(log_data=log_data) except Exception as e: log_exception(e) diff --git a/apps/api/plane/settings/common.py b/apps/api/plane/settings/common.py index 9a6ec57c93..59140c070d 100644 --- a/apps/api/plane/settings/common.py +++ b/apps/api/plane/settings/common.py @@ -402,6 +402,34 @@ WEB_URL = os.environ.get("WEB_URL") HARD_DELETE_AFTER_DAYS = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 60)) + +def _retention_days(env_var, default): + """ + Read a retention window (in days) from the environment, falling back to the + default when the variable is unset, unparseable, or negative — a negative + window would otherwise select rows with a future cutoff and delete everything. + """ + raw = os.environ.get(env_var) + if raw is None: + return default + try: + days = int(raw) + except ValueError: + return default + return days if days >= 0 else default + + +# API activity logs hold request/response payloads, so they are retained for a +# shorter window than other logs. +API_ACTIVITY_LOG_RETENTION_DAYS = _retention_days("API_ACTIVITY_LOG_RETENTION_DAYS", 14) + +# Webhook delivery logs are retained on their own window, independent of the +# generic HARD_DELETE_AFTER_DAYS. +WEBHOOK_LOG_RETENTION_DAYS = _retention_days("WEBHOOK_LOG_RETENTION_DAYS", 14) + +# Email notification logs are retained on their own window. +EMAIL_LOG_RETENTION_DAYS = _retention_days("EMAIL_LOG_RETENTION_DAYS", 7) + # Instance Changelog URL INSTANCE_CHANGELOG_URL = os.environ.get("INSTANCE_CHANGELOG_URL", "") @@ -504,7 +532,3 @@ if ENABLE_DRF_SPECTACULAR: REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema" INSTALLED_APPS.append("drf_spectacular") from .openapi import SPECTACULAR_SETTINGS # noqa: F401 - -# MongoDB Settings -MONGO_DB_URL = os.environ.get("MONGO_DB_URL", False) -MONGO_DB_DATABASE = os.environ.get("MONGO_DB_DATABASE", False) diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.py index e71f40bf14..a346eca141 100644 --- a/apps/api/plane/settings/local.py +++ b/apps/api/plane/settings/local.py @@ -75,11 +75,6 @@ LOGGING = { "handlers": ["console"], "propagate": False, }, - "plane.mongo": { - "level": "INFO", - "handlers": ["console"], - "propagate": False, - }, "plane.authentication": { "level": "INFO", "handlers": ["console"], diff --git a/apps/api/plane/settings/mongo.py b/apps/api/plane/settings/mongo.py deleted file mode 100644 index 7855a52d51..0000000000 --- a/apps/api/plane/settings/mongo.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) 2023-present Plane Software, Inc. and contributors -# SPDX-License-Identifier: AGPL-3.0-only -# See the LICENSE file for details. - -# Django imports -from django.conf import settings -import logging - -# Third party imports -from pymongo import MongoClient -from pymongo.database import Database -from pymongo.collection import Collection -from typing import Optional, TypeVar, Type - - -T = TypeVar("T", bound="MongoConnection") - -# Set up logger -logger = logging.getLogger("plane.mongo") - - -class MongoConnection: - """ - A singleton class that manages MongoDB connections. - - This class ensures only one MongoDB connection is maintained throughout the application. - It provides methods to access the MongoDB client, database, and collections. - - Attributes: - _instance (Optional[MongoConnection]): The singleton instance of this class - _client (Optional[MongoClient]): The MongoDB client instance - _db (Optional[Database]): The MongoDB database instance - """ - - _instance: Optional["MongoConnection"] = None - _client: Optional[MongoClient] = None - _db: Optional[Database] = None - - def __new__(cls: Type[T]) -> T: - """ - Creates a new instance of MongoConnection if one doesn't exist. - - Returns: - MongoConnection: The singleton instance - """ - if cls._instance is None: - cls._instance = super(MongoConnection, cls).__new__(cls) - try: - mongo_url = getattr(settings, "MONGO_DB_URL", None) - mongo_db_database = getattr(settings, "MONGO_DB_DATABASE", None) - - if not mongo_url or not mongo_db_database: - logger.warning( - "MongoDB connection parameters not configured. MongoDB functionality will be disabled." - ) - return cls._instance - - cls._client = MongoClient(mongo_url) - cls._db = cls._client[mongo_db_database] - - # Test the connection - cls._client.server_info() - logger.info("MongoDB connection established successfully") - except Exception as e: - logger.warning( - f"Failed to initialize MongoDB connection: {str(e)}. MongoDB functionality will be disabled." - ) - return cls._instance - - @classmethod - def get_client(cls) -> Optional[MongoClient]: - """ - Returns the MongoDB client instance. - - Returns: - Optional[MongoClient]: The MongoDB client instance or None if not configured - """ - if cls._client is None: - cls._instance = cls() - return cls._client - - @classmethod - def get_db(cls) -> Optional[Database]: - """ - Returns the MongoDB database instance. - - Returns: - Optional[Database]: The MongoDB database instance or None if not configured - """ - if cls._db is None: - cls._instance = cls() - return cls._db - - @classmethod - def get_collection(cls, collection_name: str) -> Optional[Collection]: - """ - Returns a MongoDB collection by name. - - Args: - collection_name (str): The name of the collection to retrieve - - Returns: - Optional[Collection]: The MongoDB collection instance or None if not configured - """ - try: - db = cls.get_db() - if db is None: - logger.warning(f"Cannot access collection '{collection_name}': MongoDB not configured") - return None - return db[collection_name] - except Exception as e: - logger.warning(f"Failed to access collection '{collection_name}': {str(e)}") - return None - - @classmethod - def is_configured(cls) -> bool: - """ - Check if MongoDB is properly configured and connected. - - Returns: - bool: True if MongoDB is configured and connected, False otherwise - """ - - if cls._client is None: - cls._instance = cls() - return cls._client is not None and cls._db is not None diff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.py index f58450a9da..1dbf43196a 100644 --- a/apps/api/plane/settings/production.py +++ b/apps/api/plane/settings/production.py @@ -85,11 +85,6 @@ LOGGING = { "handlers": ["console"], "propagate": False, }, - "plane.mongo": { - "level": "INFO", - "handlers": ["console"], - "propagate": False, - }, "plane.authentication": { "level": "DEBUG" if DEBUG else "INFO", "handlers": ["console"], diff --git a/apps/api/plane/tests/README.md b/apps/api/plane/tests/README.md index df9aba6da1..3d6936d785 100644 --- a/apps/api/plane/tests/README.md +++ b/apps/api/plane/tests/README.md @@ -91,7 +91,7 @@ When writing tests, follow these guidelines: - For web app API (`/api/`), use `session_client` - For smoke tests with real HTTP, use `plane_server` 3. Use the correct URL namespace when reverse-resolving URLs: - - For external API, use `reverse("api:endpoint_name")` + - For external API, use `reverse("api:endpoint_name")` - For web app API, use `reverse("endpoint_name")` 4. Add the `@pytest.mark.django_db` decorator to tests that interact with the database. 5. Add the appropriate markers (`@pytest.mark.contract`, etc.) to categorize tests. @@ -101,7 +101,7 @@ When writing tests, follow these guidelines: Common fixtures are defined in: - `conftest.py`: General fixtures for authentication, database access, etc. -- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery, MongoDB) +- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery) - `factories.py`: Test factories for easy model instance creation ## Best Practices @@ -125,7 +125,7 @@ When writing tests, follow these guidelines: Tests for components that interact with external services should: -1. Use the `mock_redis`, `mock_elasticsearch`, `mock_mongodb`, and `mock_celery` fixtures for unit and most contract tests. +1. Use the `mock_redis`, `mock_elasticsearch`, and `mock_celery` fixtures for unit and most contract tests. 2. For more comprehensive contract tests, use Docker-based test containers (optional). ## Coverage Reports @@ -140,4 +140,4 @@ This creates an HTML report in the `htmlcov/` directory. ## Migration from Old Tests -Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories. \ No newline at end of file +Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories. diff --git a/apps/api/plane/tests/conftest_external.py b/apps/api/plane/tests/conftest_external.py index cd5469caa6..16bc31a430 100644 --- a/apps/api/plane/tests/conftest_external.py +++ b/apps/api/plane/tests/conftest_external.py @@ -51,41 +51,6 @@ def mock_elasticsearch(): yield mock_es_client -@pytest.fixture -def mock_mongodb(): - """ - Mock MongoDB for testing without actual MongoDB connection. - - This fixture patches PyMongo to return a MagicMock that behaves like a MongoDB client. - """ - # Create mock MongoDB clients and collections - mock_mongo_client = MagicMock() - mock_mongo_db = MagicMock() - mock_mongo_collection = MagicMock() - - # Set up the chain: client -> database -> collection - mock_mongo_client.__getitem__.return_value = mock_mongo_db - mock_mongo_client.get_database.return_value = mock_mongo_db - mock_mongo_db.__getitem__.return_value = mock_mongo_collection - - # Configure common MongoDB collection operations - mock_mongo_collection.find_one.return_value = None - mock_mongo_collection.find.return_value = MagicMock(__iter__=lambda x: iter([]), count=lambda: 0) - mock_mongo_collection.insert_one.return_value = MagicMock(inserted_id="mock_id_123", acknowledged=True) - mock_mongo_collection.insert_many.return_value = MagicMock( - inserted_ids=["mock_id_123", "mock_id_456"], acknowledged=True - ) - mock_mongo_collection.update_one.return_value = MagicMock(modified_count=1, matched_count=1, acknowledged=True) - mock_mongo_collection.update_many.return_value = MagicMock(modified_count=2, matched_count=2, acknowledged=True) - mock_mongo_collection.delete_one.return_value = MagicMock(deleted_count=1, acknowledged=True) - mock_mongo_collection.delete_many.return_value = MagicMock(deleted_count=2, acknowledged=True) - mock_mongo_collection.count_documents.return_value = 0 - - # Start the patch - with patch("pymongo.MongoClient", return_value=mock_mongo_client): - yield mock_mongo_client - - @pytest.fixture def mock_celery(): """ diff --git a/apps/api/plane/tests/contract/app/test_api_token.py b/apps/api/plane/tests/contract/app/test_api_token.py index ed071b98cb..60ebd8fde8 100644 --- a/apps/api/plane/tests/contract/app/test_api_token.py +++ b/apps/api/plane/tests/contract/app/test_api_token.py @@ -366,6 +366,23 @@ class TestApiTokenEndpoint: create_api_token_for_user.refresh_from_db() assert create_api_token_for_user.user_type == 0 + @pytest.mark.django_db + def test_patch_cannot_modify_allowed_rate_limit(self, session_client, create_user, create_api_token_for_user): + """Test that allowed_rate_limit cannot be modified via PATCH""" + # Arrange + session_client.force_authenticate(user=create_user) + url = reverse("api-tokens-details", kwargs={"pk": create_api_token_for_user.pk}) + original_rate_limit = create_api_token_for_user.allowed_rate_limit + update_data = {"allowed_rate_limit": "100000/min"} + + # Act + response = session_client.patch(url, update_data, format="json") + + # Assert + assert response.status_code == status.HTTP_200_OK + create_api_token_for_user.refresh_from_db() + assert create_api_token_for_user.allowed_rate_limit == original_rate_limit + @pytest.mark.django_db def test_patch_cannot_modify_service_token(self, session_client, create_user): """Test that service tokens cannot be modified through user token endpoint""" diff --git a/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py new file mode 100644 index 0000000000..d0a85a19bc --- /dev/null +++ b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +""" +Unit tests for the log cleanup tasks. + +Verifies that API activity logs past the retention window are hard-deleted +(removed from PostgreSQL, not soft-deleted) and that fresh logs are retained. +""" + +from datetime import timedelta + +import pytest +from django.conf import settings +from django.utils import timezone + +from uuid import uuid4 + +from plane.bgtasks.cleanup_task import ( + delete_api_logs, + delete_email_notification_logs, + delete_webhook_logs, + process_cleanup_task, +) +from plane.db.models import APIActivityLog, EmailNotificationLog, WebhookLog +from plane.tests.factories import UserFactory, WorkspaceFactory + + +def _make_api_log(created_at): + log = APIActivityLog.objects.create( + token_identifier="hashed-token", + path="/api/v1/workspaces/", + method="GET", + response_code=200, + ) + # created_at is auto-set on insert, so backdate it explicitly afterwards. + APIActivityLog.all_objects.filter(pk=log.pk).update(created_at=created_at) + return log + + +def _make_webhook_log(workspace, created_at): + log = WebhookLog.objects.create( + workspace=workspace, + webhook=uuid4(), + event_type="issue", + request_method="POST", + response_status="200", + ) + WebhookLog.all_objects.filter(pk=log.pk).update(created_at=created_at) + return log + + +def _make_email_log(user, sent_at): + return EmailNotificationLog.objects.create( + receiver=user, + triggered_by=user, + entity_name="issue", + entity="issue", + sent_at=sent_at, + ) + + +@pytest.mark.unit +@pytest.mark.django_db +class TestDeleteApiLogs: + def test_expired_logs_are_hard_deleted(self): + retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS + expired = _make_api_log(timezone.now() - timedelta(days=retention_days + 1)) + + delete_api_logs() + + # Hard delete: the row must be gone even from the unfiltered manager. + assert not APIActivityLog.all_objects.filter(pk=expired.pk).exists() + + def test_recent_logs_are_retained(self): + retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS + recent = _make_api_log(timezone.now() - timedelta(days=retention_days - 1)) + + delete_api_logs() + + assert APIActivityLog.all_objects.filter(pk=recent.pk).exists() + + +@pytest.mark.unit +@pytest.mark.django_db +class TestDeleteWebhookLogs: + def test_expired_logs_are_hard_deleted(self): + workspace = WorkspaceFactory() + retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS + expired = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days + 1)) + + delete_webhook_logs() + + assert not WebhookLog.all_objects.filter(pk=expired.pk).exists() + + def test_recent_logs_are_retained(self): + workspace = WorkspaceFactory() + retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS + recent = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days - 1)) + + delete_webhook_logs() + + assert WebhookLog.all_objects.filter(pk=recent.pk).exists() + + +@pytest.mark.unit +@pytest.mark.django_db +class TestDeleteEmailLogs: + def test_expired_logs_are_hard_deleted(self): + user = UserFactory() + retention_days = settings.EMAIL_LOG_RETENTION_DAYS + expired = _make_email_log(user, timezone.now() - timedelta(days=retention_days + 1)) + + delete_email_notification_logs() + + assert not EmailNotificationLog.all_objects.filter(pk=expired.pk).exists() + + def test_recent_logs_are_retained(self): + user = UserFactory() + retention_days = settings.EMAIL_LOG_RETENTION_DAYS + recent = _make_email_log(user, timezone.now() - timedelta(days=retention_days - 1)) + + delete_email_notification_logs() + + assert EmailNotificationLog.all_objects.filter(pk=recent.pk).exists() + + +@pytest.mark.unit +class TestProcessCleanupTaskErrorHandling: + def test_batch_delete_failure_is_swallowed(self): + """A failing batch is logged and skipped; the run does not raise.""" + + class _BoomManager: + @staticmethod + def filter(**kwargs): + raise RuntimeError("db unavailable") + + class _BoomModel: + all_objects = _BoomManager() + + # Should not raise even though the delete blows up. + process_cleanup_task(lambda: iter([1, 2, 3]), _BoomModel, "Boom") diff --git a/apps/api/plane/tests/unit/middleware/test_logger.py b/apps/api/plane/tests/unit/middleware/test_logger.py new file mode 100644 index 0000000000..5c13f53f6f --- /dev/null +++ b/apps/api/plane/tests/unit/middleware/test_logger.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +""" +Unit tests for APITokenLogMiddleware. + +Covers the credential-hygiene guarantees of the external API request logger: +- the raw API key is never persisted (a non-reversible hash is stored instead) +- sensitive request headers are redacted before being logged +""" + +import hashlib +import hmac +from unittest.mock import Mock, patch + +import pytest +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse +from django.test import RequestFactory + +from plane.middleware.logger import APITokenLogMiddleware + + +@pytest.fixture +def request_factory(): + return RequestFactory() + + +@pytest.fixture +def middleware(): + return APITokenLogMiddleware(Mock(return_value=HttpResponse(b"{}"))) + + +@pytest.mark.unit +class TestAPITokenLogMiddleware: + API_KEY = "plane_api_supersecretvalue" + AUTHORIZATION = "Bearer secret-bearer-token" + COOKIE = "sessionid=secret-session-value" + + def _captured_log_data(self, middleware, request_factory): + request = request_factory.get( + "/api/v1/workspaces/", + HTTP_X_API_KEY=self.API_KEY, + HTTP_AUTHORIZATION=self.AUTHORIZATION, + HTTP_COOKIE=self.COOKIE, + ) + request.user = AnonymousUser() + response = HttpResponse(b"{}") + with patch("plane.middleware.logger.process_logs") as process_logs: + middleware.process_request(request, response, request_body=b"") + assert process_logs.delay.called + return process_logs.delay.call_args.kwargs["log_data"] + + def test_token_identifier_is_hashed_not_plaintext(self, middleware, request_factory): + log_data = self._captured_log_data(middleware, request_factory) + + expected_hash = hmac.new( + settings.SECRET_KEY.encode(), self.API_KEY.encode(), hashlib.sha256 + ).hexdigest() + assert log_data["token_identifier"] == expected_hash + assert self.API_KEY not in log_data["token_identifier"] + + def test_sensitive_headers_are_redacted(self, middleware, request_factory): + log_data = self._captured_log_data(middleware, request_factory) + + # None of the sensitive header values may appear in the logged headers. + assert self.API_KEY not in log_data["headers"] + assert self.AUTHORIZATION not in log_data["headers"] + assert self.COOKIE not in log_data["headers"] + assert "[REDACTED]" in log_data["headers"] + + def test_no_log_without_api_key(self, middleware, request_factory): + request = request_factory.get("/api/v1/workspaces/") + request.user = AnonymousUser() + with patch("plane.middleware.logger.process_logs") as process_logs: + middleware.process_request(request, HttpResponse(b"{}"), request_body=b"") + assert not process_logs.delay.called diff --git a/apps/api/plane/tests/unit/settings/test_retention.py b/apps/api/plane/tests/unit/settings/test_retention.py new file mode 100644 index 0000000000..10bc79192d --- /dev/null +++ b/apps/api/plane/tests/unit/settings/test_retention.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Unit tests for the log-retention env parsing helper.""" + +import pytest + +from plane.settings.common import _retention_days + +ENV_VAR = "TEST_RETENTION_DAYS" + + +@pytest.mark.unit +class TestRetentionDays: + def test_uses_default_when_unset(self, monkeypatch): + monkeypatch.delenv(ENV_VAR, raising=False) + assert _retention_days(ENV_VAR, 14) == 14 + + def test_uses_env_value_when_valid(self, monkeypatch): + monkeypatch.setenv(ENV_VAR, "30") + assert _retention_days(ENV_VAR, 14) == 30 + + def test_zero_is_allowed(self, monkeypatch): + monkeypatch.setenv(ENV_VAR, "0") + assert _retention_days(ENV_VAR, 14) == 0 + + def test_negative_falls_back_to_default(self, monkeypatch): + monkeypatch.setenv(ENV_VAR, "-5") + assert _retention_days(ENV_VAR, 14) == 14 + + def test_unparseable_falls_back_to_default(self, monkeypatch): + monkeypatch.setenv(ENV_VAR, "abc") + assert _retention_days(ENV_VAR, 7) == 7 diff --git a/apps/api/requirements/base.txt b/apps/api/requirements/base.txt index a34bf6de11..68c4a4a43c 100644 --- a/apps/api/requirements/base.txt +++ b/apps/api/requirements/base.txt @@ -9,8 +9,6 @@ psycopg==3.3.0 psycopg-binary==3.3.0 psycopg-c==3.3.0 dj-database-url==2.1.0 -# mongo -pymongo==4.6.3 # redis redis==5.0.4 django-redis==5.4.0