references/plane
1
0
Fork 0
mirror of https://github.com/makeplane/plane.git synced 2026-06-13 19:19:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: logging with retention + API token hardening (#9148)

Browse Source 
* fix: harden API token handling against rate-limit tampering and plaintext logging

- Make `allowed_rate_limit` read-only on APITokenSerializer so users can no
  longer raise their own API token rate limit via PATCH (GHSA-xfgr-2x3f-g2cf).
- Stop persisting API keys in plaintext in APITokenLogMiddleware: store a
  SHA-256 hash as the token identifier and redact sensitive request headers
  (X-Api-Key, Authorization, Cookie) before logging (GHSA-r5p8-cj3q-38cc).

* refactor: remove MongoDB log sink and add per-log-type retention

Logs are now written to and cleared from PostgreSQL only; MongoDB is no
longer used as a log sink or archive.

- Drop the MongoDB write/archival paths from the API request logger, the
  webhook log writer, and the cleanup tasks; Postgres is the sole sink.
- Cleanup tasks now hard-delete expired rows in batches via `all_objects`
  (rows are removed immediately, not soft-deleted).
- Add env-backed, per-log-type retention settings: API activity logs
  (API_ACTIVITY_LOG_RETENTION_DAYS, default 14), webhook logs
  (WEBHOOK_LOG_RETENTION_DAYS, default 14), email logs
  (EMAIL_LOG_RETENTION_DAYS, default 7). HARD_DELETE_AFTER_DAYS no longer
  drives any log cleanup.
- Delete settings/mongo.py, remove MONGO_DB_* settings and the plane.mongo
  loggers, and drop the pymongo dependency.

* chore: gitignore local advisories.md notes file

* fix: use keyed HMAC-SHA256 for API token log identifier

Address CodeQL "weak hashing of sensitive data" by hashing the API key with
a SECRET_KEY-keyed HMAC instead of a bare SHA-256. The identifier is a
non-reversible tokenization of a high-entropy key (not password storage);
keying it also prevents precomputing the digest from a known key value.

* chore: address review feedback on log cleanup and request logging

- process_logs accepts extra kwargs so jobs enqueued by an older release
  (with a mongo_log arg) don't fail during a rolling deploy.
- Log-cleanup batch delete failures are logged and skipped rather than
  aborting the run, so a single bad batch can't block the rest.
- Extend logger middleware test to assert Authorization and Cookie headers
  are redacted; add a test that a failing cleanup batch is swallowed.

* fix: fall back to default when a log retention env value is invalid

Negative (or unparseable) retention values would compute a future cutoff and
delete every log row. The retention settings now fall back to their defaults
in that case via a shared `_retention_days` helper.
This commit is contained in:
sriram veeraghanta
2026-05-27 16:00:05 +05:30
committed by GitHub
parent 310d2eda21
commit edf2475413
17 changed files with 398 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -114,3 +114,6 @@ scripts/
# i18n auto-generated types (regenerated on every build)
packages/i18n/src/types/keys.generated.ts
# Local security advisory notes (not for version control)
/advisories.md
apps/api/plane/app/serializers/api.py
+1
View File
@@ -22,6 +22,7 @@ class APITokenSerializer(BaseSerializer):
"is_active",
"last_used",
"user_type",
"allowed_rate_limit",
]
apps/api/plane/bgtasks/cleanup_task.py
+47 -318
View File
@@ -5,19 +5,16 @@
# Python imports
from datetime import timedelta
import logging
from typing import List, Dict, Any, Callable, Optional
import os
from typing import Callable, Iterable
# Django imports
from django.conf import settings
from django.utils import timezone
from django.db.models import F, Window, Subquery
from django.db.models.functions import RowNumber
# Third party imports
from celery import shared_task
from pymongo.errors import BulkWriteError
from pymongo.collection import Collection
from pymongo.operations import InsertOne
# Module imports
from plane.db.models import (
@@ -27,7 +24,6 @@ from plane.db.models import (
IssueDescriptionVersion,
WebhookLog,
)
from plane.settings.mongo import MongoConnection
from plane.utils.exception_logger import log_exception
@@ -35,285 +31,75 @@ logger = logging.getLogger("plane.worker")
BATCH_SIZE = 500
def get_mongo_collection(collection_name: str) -> Optional[Collection]:
"""Get MongoDB collection if available, otherwise return None."""
if not MongoConnection.is_configured():
logger.info("MongoDB not configured")
return None
try:
mongo_collection = MongoConnection.get_collection(collection_name)
logger.info(f"MongoDB collection '{collection_name}' connected successfully")
return mongo_collection
except Exception as e:
logger.error(f"Failed to get MongoDB collection: {str(e)}")
log_exception(e)
return None
def flush_to_mongo_and_delete(
mongo_collection: Optional[Collection],
buffer: List[Dict[str, Any]],
ids_to_delete: List[int],
model,
mongo_available: bool,
) -> None:
"""
Inserts a batch of records into MongoDB and deletes the corresponding rows from PostgreSQL.
"""
if not buffer:
logger.debug("No records to flush - buffer is empty")
return
logger.info(f"Starting batch flush: {len(buffer)} records, {len(ids_to_delete)} IDs to delete")
mongo_archival_failed = False
# Try to insert into MongoDB if available
if mongo_collection is not None and mongo_available:
try:
mongo_collection.bulk_write([InsertOne(doc) for doc in buffer])
except BulkWriteError as bwe:
logger.error(f"MongoDB bulk write error: {str(bwe)}")
log_exception(bwe)
mongo_archival_failed = True
# If MongoDB is available and archival failed, log the error and return
if mongo_available and mongo_archival_failed:
logger.error(f"MongoDB archival failed for {len(buffer)} records")
return
# Delete from PostgreSQL - delete() returns (count, {model: count})
delete_result = model.all_objects.filter(id__in=ids_to_delete).delete()
deleted_count = delete_result[0] if delete_result and isinstance(delete_result, tuple) else 0
logger.info(f"Batch flush completed: {deleted_count} records deleted")
def process_cleanup_task(
queryset_func: Callable,
transform_func: Callable[[Dict], Dict],
queryset_func: Callable[[], Iterable],
model,
task_name: str,
collection_name: str,
):
"""
Generic function to process cleanup tasks.
Batch-delete expired rows for the given model from PostgreSQL.
Args:
queryset_func: Function that returns the queryset to process
transform_func: Function to transform each record for MongoDB
model: Django model class
task_name: Name of the task for logging
collection_name: MongoDB collection name
queryset_func: Callable returning an iterable of primary keys to delete.
model: Django model class.
task_name: Name of the task for logging.
"""
logger.info(f"Starting {task_name} cleanup task")
# Get MongoDB collection
mongo_collection = get_mongo_collection(collection_name)
mongo_available = mongo_collection is not None
# Get queryset
queryset = queryset_func()
# Process records in batches
buffer: List[Dict[str, Any]] = []
ids_to_delete: List[int] = []
total_processed = 0
total_deleted = 0
total_batches = 0
batch: list = []
for record in queryset:
# Transform record for MongoDB
buffer.append(transform_func(record))
ids_to_delete.append(record["id"])
# Flush batch when it reaches BATCH_SIZE
if len(buffer) >= BATCH_SIZE:
total_batches += 1
flush_to_mongo_and_delete(
mongo_collection=mongo_collection,
buffer=buffer,
ids_to_delete=ids_to_delete,
model=model,
mongo_available=mongo_available,
)
total_processed += len(buffer)
buffer.clear()
ids_to_delete.clear()
# Process final batch if any records remain
if buffer:
def flush(ids: list) -> None:
nonlocal total_deleted, total_batches
if not ids:
return
total_batches += 1
flush_to_mongo_and_delete(
mongo_collection=mongo_collection,
buffer=buffer,
ids_to_delete=ids_to_delete,
model=model,
mongo_available=mongo_available,
)
total_processed += len(buffer)
try:
# `all_objects` is a plain manager, so this is a hard delete — rows
# are removed from PostgreSQL immediately rather than soft-deleted.
delete_result = model.all_objects.filter(id__in=ids).delete()
deleted = delete_result[0] if isinstance(delete_result, tuple) else 0
total_deleted += deleted
except Exception as e:
# Log and skip a failed batch rather than aborting the whole run, so
# a single bad batch doesn't block cleanup of the remaining rows.
log_exception(e)
for record_id in queryset_func():
batch.append(record_id)
if len(batch) >= BATCH_SIZE:
flush(batch)
batch = []
# Flush the final partial batch
flush(batch)
logger.info(
f"{task_name} cleanup task completed",
extra={
"total_records_processed": total_processed,
"total_batches": total_batches,
"mongo_available": mongo_available,
"collection_name": collection_name,
},
extra={"total_records_deleted": total_deleted, "total_batches": total_batches},
)
# Transform functions for each model
def transform_api_log(record: Dict) -> Dict:
"""Transform API activity log record."""
return {
"id": str(record["id"]),
"created_at": str(record["created_at"]) if record.get("created_at") else None,
"token_identifier": str(record["token_identifier"]),
"path": record["path"],
"method": record["method"],
"query_params": record.get("query_params"),
"headers": record.get("headers"),
"body": record.get("body"),
"response_code": record["response_code"],
"response_body": record["response_body"],
"ip_address": record["ip_address"],
"user_agent": record["user_agent"],
"created_by_id": str(record["created_by_id"]),
}
def transform_email_log(record: Dict) -> Dict:
"""Transform email notification log record."""
return {
"id": str(record["id"]),
"created_at": str(record["created_at"]) if record.get("created_at") else None,
"receiver_id": str(record["receiver_id"]),
"triggered_by_id": str(record["triggered_by_id"]),
"entity_identifier": str(record["entity_identifier"]),
"entity_name": record["entity_name"],
"data": record["data"],
"processed_at": (str(record["processed_at"]) if record.get("processed_at") else None),
"sent_at": str(record["sent_at"]) if record.get("sent_at") else None,
"entity": record["entity"],
"old_value": str(record["old_value"]),
"new_value": str(record["new_value"]),
"created_by_id": str(record["created_by_id"]),
}
def transform_page_version(record: Dict) -> Dict:
"""Transform page version record."""
return {
"id": str(record["id"]),
"created_at": str(record["created_at"]) if record.get("created_at") else None,
"page_id": str(record["page_id"]),
"workspace_id": str(record["workspace_id"]),
"owned_by_id": str(record["owned_by_id"]),
"description_html": record["description_html"],
"description_binary": record["description_binary"],
"description_stripped": record["description_stripped"],
"description_json": record["description_json"],
"sub_pages_data": record["sub_pages_data"],
"created_by_id": str(record["created_by_id"]),
"updated_by_id": str(record["updated_by_id"]),
"deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,
"last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),
}
def transform_issue_description_version(record: Dict) -> Dict:
"""Transform issue description version record."""
return {
"id": str(record["id"]),
"created_at": str(record["created_at"]) if record.get("created_at") else None,
"issue_id": str(record["issue_id"]),
"workspace_id": str(record["workspace_id"]),
"project_id": str(record["project_id"]),
"created_by_id": str(record["created_by_id"]),
"updated_by_id": str(record["updated_by_id"]),
"owned_by_id": str(record["owned_by_id"]),
"last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),
"description_binary": record["description_binary"],
"description_html": record["description_html"],
"description_stripped": record["description_stripped"],
"description_json": record["description_json"],
"deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,
}
def transform_webhook_log(record: Dict):
"""Transfer webhook logs to a new destination."""
return {
"id": str(record["id"]),
"created_at": str(record["created_at"]) if record.get("created_at") else None,
"workspace_id": str(record["workspace_id"]),
"webhook": str(record["webhook"]),
# Request
"event_type": str(record["event_type"]),
"request_method": str(record["request_method"]),
"request_headers": str(record["request_headers"]),
"request_body": str(record["request_body"]),
# Response
"response_status": str(record["response_status"]),
"response_body": str(record["response_body"]),
"response_headers": str(record["response_headers"]),
# retry count
"retry_count": str(record["retry_count"]),
}
# Queryset functions for each cleanup task
# Queryset functions for each cleanup task — each yields primary keys to delete
def get_api_logs_queryset():
"""Get API logs older than cutoff days."""
cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
cutoff_time = timezone.now() - timedelta(days=cutoff_days)
"""Get API activity logs older than the API retention window."""
cutoff_time = timezone.now() - timedelta(days=settings.API_ACTIVITY_LOG_RETENTION_DAYS)
logger.info(f"API logs cutoff time: {cutoff_time}")
return (
APIActivityLog.all_objects.filter(created_at__lte=cutoff_time)
.values(
"id",
"created_at",
"token_identifier",
"path",
"method",
"query_params",
"headers",
"body",
"response_code",
"response_body",
"ip_address",
"user_agent",
"created_by_id",
)
.values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
def get_email_logs_queryset():
"""Get email logs older than cutoff days."""
cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
cutoff_time = timezone.now() - timedelta(days=cutoff_days)
"""Get email logs older than the email retention window."""
cutoff_time = timezone.now() - timedelta(days=settings.EMAIL_LOG_RETENTION_DAYS)
logger.info(f"Email logs cutoff time: {cutoff_time}")
return (
EmailNotificationLog.all_objects.filter(sent_at__lte=cutoff_time)
.values(
"id",
"created_at",
"receiver_id",
"triggered_by_id",
"entity_identifier",
"entity_name",
"data",
"processed_at",
"sent_at",
"entity",
"old_value",
"new_value",
"created_by_id",
)
.values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
@@ -334,22 +120,7 @@ def get_page_versions_queryset():
return (
PageVersion.all_objects.filter(id__in=Subquery(subq))
.values(
"id",
"created_at",
"page_id",
"workspace_id",
"owned_by_id",
"description_html",
"description_binary",
"description_stripped",
"description_json",
"sub_pages_data",
"created_by_id",
"updated_by_id",
"deleted_at",
"last_saved_at",
)
.values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
@@ -370,52 +141,20 @@ def get_issue_description_versions_queryset():
return (
IssueDescriptionVersion.all_objects.filter(id__in=Subquery(subq))
.values(
"id",
"created_at",
"issue_id",
"workspace_id",
"project_id",
"created_by_id",
"updated_by_id",
"owned_by_id",
"last_saved_at",
"description_binary",
"description_html",
"description_stripped",
"description_json",
"deleted_at",
)
.values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
def get_webhook_logs_queryset():
"""Get email logs older than cutoff days."""
cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
cutoff_time = timezone.now() - timedelta(days=cutoff_days)
"""Get webhook logs older than the webhook retention window."""
cutoff_time = timezone.now() - timedelta(days=settings.WEBHOOK_LOG_RETENTION_DAYS)
logger.info(f"Webhook logs cutoff time: {cutoff_time}")
return (
WebhookLog.all_objects.filter(created_at__lte=cutoff_time)
.values(
"id",
"created_at",
"workspace_id",
"webhook",
"event_type",
# Request
"request_method",
"request_headers",
"request_body",
# Response
"response_status",
"response_body",
"response_headers",
"retry_count",
)
.order_by("created_at")
.iterator(chunk_size=100)
.values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
@@ -424,10 +163,8 @@ def delete_api_logs():
"""Delete old API activity logs."""
process_cleanup_task(
queryset_func=get_api_logs_queryset,
transform_func=transform_api_log,
model=APIActivityLog,
task_name="API Activity Log",
collection_name="api_activity_logs",
)
@@ -436,10 +173,8 @@ def delete_email_notification_logs():
"""Delete old email notification logs."""
process_cleanup_task(
queryset_func=get_email_logs_queryset,
transform_func=transform_email_log,
model=EmailNotificationLog,
task_name="Email Notification Log",
collection_name="email_notification_logs",
)
@@ -448,10 +183,8 @@ def delete_page_versions():
"""Delete excess page versions."""
process_cleanup_task(
queryset_func=get_page_versions_queryset,
transform_func=transform_page_version,
model=PageVersion,
task_name="Page Version",
collection_name="page_versions",
)
@@ -460,20 +193,16 @@ def delete_issue_description_versions():
"""Delete excess issue description versions."""
process_cleanup_task(
queryset_func=get_issue_description_versions_queryset,
transform_func=transform_issue_description_version,
model=IssueDescriptionVersion,
task_name="Issue Description Version",
collection_name="issue_description_versions",
)
@shared_task
def delete_webhook_logs():
"""Delete old webhook logs"""
"""Delete old webhook logs."""
process_cleanup_task(
queryset_func=get_webhook_logs_queryset,
transform_func=transform_webhook_log,
model=WebhookLog,
task_name="Webhook Log",
collection_name="webhook_logs",
)
apps/api/plane/bgtasks/logger_task.py
+9 -68
View File
@@ -4,14 +4,12 @@
# Python imports
import logging
from typing import Optional, Dict, Any
from typing import Dict, Any
# Third party imports
from pymongo.collection import Collection
from celery import shared_task
# Django imports
from plane.settings.mongo import MongoConnection
from plane.utils.exception_logger import log_exception
from plane.db.models import APIActivityLog
@@ -19,66 +17,9 @@ from plane.db.models import APIActivityLog
logger = logging.getLogger("plane.worker")
def get_mongo_collection() -> Optional[Collection]:
"""
Returns the MongoDB collection for external API activity logs.
"""
if not MongoConnection.is_configured():
logger.info("MongoDB not configured")
return None
try:
return MongoConnection.get_collection("api_activity_logs")
except Exception as e:
logger.error(f"Error getting MongoDB collection: {str(e)}")
log_exception(e)
return None
def safe_decode_body(content: bytes) -> Optional[str]:
"""
Safely decodes request/response body content, handling binary data.
Returns "[Binary Content]" if the content is binary, or a string representation of the content.
Returns None if the content is None or empty.
"""
# If the content is None, return None
if content is None:
return None
# If the content is an empty bytes object, return None
if content == b"":
return None
# Check if content is binary by looking for common binary file signatures
if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"):
return "[Binary Content]"
try:
return content.decode("utf-8")
except UnicodeDecodeError:
return "[Could not decode content]"
def log_to_mongo(log_document: Dict[str, Any]) -> bool:
"""
Logs the request to MongoDB if available.
"""
mongo_collection = get_mongo_collection()
if mongo_collection is None:
logger.error("MongoDB not configured")
return False
try:
mongo_collection.insert_one(log_document)
return True
except Exception as e:
log_exception(e)
return False
def log_to_postgres(log_data: Dict[str, Any]) -> bool:
"""
Fallback to logging to PostgreSQL if MongoDB is unavailable.
Persist an external API request log to PostgreSQL.
"""
try:
APIActivityLog.objects.create(**log_data)
@@ -89,12 +30,12 @@ def log_to_postgres(log_data: Dict[str, Any]) -> bool:
@shared_task
def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]) -> None:
"""
Process logs to save to MongoDB or Postgres based on the configuration
def process_logs(log_data: Dict[str, Any], **_: Any) -> None:
"""
Persist external API request logs to PostgreSQL.
if MongoConnection.is_configured():
log_to_mongo(mongo_log)
else:
log_to_postgres(log_data)
The catch-all kwargs keep this task signature compatible with jobs enqueued
by an older release (which passed a `mongo_log` argument), so in-flight tasks
don't fail during a rolling deploy. It can be dropped once no such jobs remain.
"""
log_to_postgres(log_data)
apps/api/plane/bgtasks/webhook_task.py
+6 -25
View File
@@ -53,7 +53,6 @@ from plane.license.utils.instance_value import get_email_configuration
from plane.utils.email import generate_plain_text_from_html
from plane.utils.exception_logger import log_exception
from plane.utils.ip_address import validate_url
from plane.settings.mongo import MongoConnection
SERIALIZER_MAPPER = {
@@ -102,9 +101,6 @@ def save_webhook_log(
retry_count: int,
event_type: str,
) -> None:
# webhook_logs
mongo_collection = MongoConnection.get_collection("webhook_logs")
log_data = {
"workspace_id": str(webhook.workspace_id),
"webhook": str(webhook.id),
@@ -118,27 +114,12 @@ def save_webhook_log(
"retry_count": retry_count,
}
mongo_save_success = False
if mongo_collection is not None:
try:
# insert the log data into the mongo collection
mongo_collection.insert_one(log_data)
logger.info("Webhook log saved successfully to mongo")
mongo_save_success = True
except Exception as e:
log_exception(e, warning=True)
logger.error(f"Failed to save webhook log: {e}")
mongo_save_success = False
# if the mongo save is not successful, save the log data into the database
if not mongo_save_success:
try:
# insert the log data into the database
WebhookLog.objects.create(**log_data)
logger.info("Webhook log saved successfully to database")
except Exception as e:
log_exception(e, warning=True)
logger.error(f"Failed to save webhook log: {e}")
try:
WebhookLog.objects.create(**log_data)
logger.info("Webhook log saved successfully to database")
except Exception as e:
log_exception(e, warning=True)
logger.error(f"Failed to save webhook log: {e}")
def get_model_data(event: str, event_id: Union[str, List[str]], many: bool = False) -> Dict[str, Any]:
apps/api/plane/middleware/logger.py
+27 -18
View File
@@ -3,12 +3,14 @@
# See the LICENSE file for details.
# Python imports
import hashlib
import hmac
import logging
import time
# Django imports
from django.conf import settings
from django.http import HttpRequest
from django.utils import timezone
# Third party imports
from rest_framework.request import Request
@@ -77,7 +79,7 @@ class RequestLoggerMiddleware:
class APITokenLogMiddleware:
"""
Middleware to log External API requests to MongoDB or PostgreSQL.
Middleware to log External API requests to PostgreSQL.
"""
def __init__(self, get_response):
@@ -111,6 +113,20 @@ class APITokenLogMiddleware:
except UnicodeDecodeError:
return "[Could not decode content]"
# Headers whose values must never be persisted in plaintext logs
SENSITIVE_HEADERS = frozenset({"x-api-key", "authorization", "cookie"})
def _redacted_headers(self, request):
"""
Returns the request headers as a string with sensitive values redacted,
so that credentials such as the API key are never stored in plaintext.
"""
redacted = {
key: ("[REDACTED]" if key.lower() in self.SENSITIVE_HEADERS else value)
for key, value in request.headers.items()
}
return str(redacted)
def process_request(self, request, response, request_body):
api_key_header = "X-Api-Key"
api_key = request.headers.get(api_key_header)
@@ -121,32 +137,25 @@ class APITokenLogMiddleware:
try:
log_data = {
"token_identifier": api_key,
# Tokenize the (high-entropy) API key into a stable, non-reversible
# identifier so logs can be correlated to a token without ever
# persisting the raw key. A keyed HMAC is used rather than a bare
# hash so the digest cannot be precomputed from a known key value.
"token_identifier": hmac.new(
settings.SECRET_KEY.encode(), api_key.encode(), hashlib.sha256
).hexdigest(),
"path": request.path,
"method": request.method,
"query_params": request.META.get("QUERY_STRING", ""),
"headers": str(request.headers),
"headers": self._redacted_headers(request),
"body": self._safe_decode_body(request_body) if request_body else None,
"response_body": self._safe_decode_body(response.content) if response.content else None,
"response_code": response.status_code,
"ip_address": get_client_ip(request=request),
"user_agent": request.META.get("HTTP_USER_AGENT", None),
}
user_id = (
str(request.user.id)
if getattr(request, "user") and getattr(request.user, "is_authenticated", False)
else None
)
# Additional fields for MongoDB
mongo_log = {
**log_data,
"created_at": timezone.now(),
"updated_at": timezone.now(),
"created_by": user_id,
"updated_by": user_id,
}
process_logs.delay(log_data=log_data, mongo_log=mongo_log)
process_logs.delay(log_data=log_data)
except Exception as e:
log_exception(e)
apps/api/plane/settings/common.py
+28 -4
View File
@@ -402,6 +402,34 @@ WEB_URL = os.environ.get("WEB_URL")
HARD_DELETE_AFTER_DAYS = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 60))
def _retention_days(env_var, default):
"""
Read a retention window (in days) from the environment, falling back to the
default when the variable is unset, unparseable, or negative — a negative
window would otherwise select rows with a future cutoff and delete everything.
"""
raw = os.environ.get(env_var)
if raw is None:
return default
try:
days = int(raw)
except ValueError:
return default
return days if days >= 0 else default
# API activity logs hold request/response payloads, so they are retained for a
# shorter window than other logs.
API_ACTIVITY_LOG_RETENTION_DAYS = _retention_days("API_ACTIVITY_LOG_RETENTION_DAYS", 14)
# Webhook delivery logs are retained on their own window, independent of the
# generic HARD_DELETE_AFTER_DAYS.
WEBHOOK_LOG_RETENTION_DAYS = _retention_days("WEBHOOK_LOG_RETENTION_DAYS", 14)
# Email notification logs are retained on their own window.
EMAIL_LOG_RETENTION_DAYS = _retention_days("EMAIL_LOG_RETENTION_DAYS", 7)
# Instance Changelog URL
INSTANCE_CHANGELOG_URL = os.environ.get("INSTANCE_CHANGELOG_URL", "")
@@ -504,7 +532,3 @@ if ENABLE_DRF_SPECTACULAR:
REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema"
INSTALLED_APPS.append("drf_spectacular")
from .openapi import SPECTACULAR_SETTINGS # noqa: F401
# MongoDB Settings
MONGO_DB_URL = os.environ.get("MONGO_DB_URL", False)
MONGO_DB_DATABASE = os.environ.get("MONGO_DB_DATABASE", False)
apps/api/plane/settings/local.py
-5
View File
@@ -75,11 +75,6 @@ LOGGING = {
"handlers": ["console"],
"propagate": False,
},
"plane.mongo": {
"level": "INFO",
"handlers": ["console"],
"propagate": False,
},
"plane.authentication": {
"level": "INFO",
"handlers": ["console"],
apps/api/plane/settings/mongo.py
-126
View File
@@ -1,126 +0,0 @@
# Copyright (c) 2023-present Plane Software, Inc. and contributors
# SPDX-License-Identifier: AGPL-3.0-only
# See the LICENSE file for details.
# Django imports
from django.conf import settings
import logging
# Third party imports
from pymongo import MongoClient
from pymongo.database import Database
from pymongo.collection import Collection
from typing import Optional, TypeVar, Type
T = TypeVar("T", bound="MongoConnection")
# Set up logger
logger = logging.getLogger("plane.mongo")
class MongoConnection:
"""
A singleton class that manages MongoDB connections.
This class ensures only one MongoDB connection is maintained throughout the application.
It provides methods to access the MongoDB client, database, and collections.
Attributes:
_instance (Optional[MongoConnection]): The singleton instance of this class
_client (Optional[MongoClient]): The MongoDB client instance
_db (Optional[Database]): The MongoDB database instance
"""
_instance: Optional["MongoConnection"] = None
_client: Optional[MongoClient] = None
_db: Optional[Database] = None
def __new__(cls: Type[T]) -> T:
"""
Creates a new instance of MongoConnection if one doesn't exist.
Returns:
MongoConnection: The singleton instance
"""
if cls._instance is None:
cls._instance = super(MongoConnection, cls).__new__(cls)
try:
mongo_url = getattr(settings, "MONGO_DB_URL", None)
mongo_db_database = getattr(settings, "MONGO_DB_DATABASE", None)
if not mongo_url or not mongo_db_database:
logger.warning(
"MongoDB connection parameters not configured. MongoDB functionality will be disabled."
)
return cls._instance
cls._client = MongoClient(mongo_url)
cls._db = cls._client[mongo_db_database]
# Test the connection
cls._client.server_info()
logger.info("MongoDB connection established successfully")
except Exception as e:
logger.warning(
f"Failed to initialize MongoDB connection: {str(e)}. MongoDB functionality will be disabled."
)
return cls._instance
@classmethod
def get_client(cls) -> Optional[MongoClient]:
"""
Returns the MongoDB client instance.
Returns:
Optional[MongoClient]: The MongoDB client instance or None if not configured
"""
if cls._client is None:
cls._instance = cls()
return cls._client
@classmethod
def get_db(cls) -> Optional[Database]:
"""
Returns the MongoDB database instance.
Returns:
Optional[Database]: The MongoDB database instance or None if not configured
"""
if cls._db is None:
cls._instance = cls()
return cls._db
@classmethod
def get_collection(cls, collection_name: str) -> Optional[Collection]:
"""
Returns a MongoDB collection by name.
Args:
collection_name (str): The name of the collection to retrieve
Returns:
Optional[Collection]: The MongoDB collection instance or None if not configured
"""
try:
db = cls.get_db()
if db is None:
logger.warning(f"Cannot access collection '{collection_name}': MongoDB not configured")
return None
return db[collection_name]
except Exception as e:
logger.warning(f"Failed to access collection '{collection_name}': {str(e)}")
return None
@classmethod
def is_configured(cls) -> bool:
"""
Check if MongoDB is properly configured and connected.
Returns:
bool: True if MongoDB is configured and connected, False otherwise
"""
if cls._client is None:
cls._instance = cls()
return cls._client is not None and cls._db is not None
apps/api/plane/settings/production.py
-5
View File
@@ -85,11 +85,6 @@ LOGGING = {
"handlers": ["console"],
"propagate": False,
},
"plane.mongo": {
"level": "INFO",
"handlers": ["console"],
"propagate": False,
},
"plane.authentication": {
"level": "DEBUG" if DEBUG else "INFO",
"handlers": ["console"],
apps/api/plane/tests/README.md
+4 -4
View File
@@ -91,7 +91,7 @@ When writing tests, follow these guidelines:
- For web app API (`/api/`), use `session_client`
- For smoke tests with real HTTP, use `plane_server`
3. Use the correct URL namespace when reverse-resolving URLs:
- For external API, use `reverse("api:endpoint_name")`
- For external API, use `reverse("api:endpoint_name")`
- For web app API, use `reverse("endpoint_name")`
4. Add the `@pytest.mark.django_db` decorator to tests that interact with the database.
5. Add the appropriate markers (`@pytest.mark.contract`, etc.) to categorize tests.
@@ -101,7 +101,7 @@ When writing tests, follow these guidelines:
Common fixtures are defined in:
- `conftest.py`: General fixtures for authentication, database access, etc.
- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery, MongoDB)
- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery)
- `factories.py`: Test factories for easy model instance creation
## Best Practices
@@ -125,7 +125,7 @@ When writing tests, follow these guidelines:
Tests for components that interact with external services should:
1. Use the `mock_redis`, `mock_elasticsearch`, `mock_mongodb`, and `mock_celery` fixtures for unit and most contract tests.
1. Use the `mock_redis`, `mock_elasticsearch`, and `mock_celery` fixtures for unit and most contract tests.
2. For more comprehensive contract tests, use Docker-based test containers (optional).
## Coverage Reports
@@ -140,4 +140,4 @@ This creates an HTML report in the `htmlcov/` directory.
## Migration from Old Tests
Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.
Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.
apps/api/plane/tests/conftest_external.py
-35
View File
@@ -51,41 +51,6 @@ def mock_elasticsearch():
yield mock_es_client
@pytest.fixture
def mock_mongodb():
"""
Mock MongoDB for testing without actual MongoDB connection.
This fixture patches PyMongo to return a MagicMock that behaves like a MongoDB client.
"""
# Create mock MongoDB clients and collections
mock_mongo_client = MagicMock()
mock_mongo_db = MagicMock()
mock_mongo_collection = MagicMock()
# Set up the chain: client -> database -> collection
mock_mongo_client.__getitem__.return_value = mock_mongo_db
mock_mongo_client.get_database.return_value = mock_mongo_db
mock_mongo_db.__getitem__.return_value = mock_mongo_collection
# Configure common MongoDB collection operations
mock_mongo_collection.find_one.return_value = None
mock_mongo_collection.find.return_value = MagicMock(__iter__=lambda x: iter([]), count=lambda: 0)
mock_mongo_collection.insert_one.return_value = MagicMock(inserted_id="mock_id_123", acknowledged=True)
mock_mongo_collection.insert_many.return_value = MagicMock(
inserted_ids=["mock_id_123", "mock_id_456"], acknowledged=True
)
mock_mongo_collection.update_one.return_value = MagicMock(modified_count=1, matched_count=1, acknowledged=True)
mock_mongo_collection.update_many.return_value = MagicMock(modified_count=2, matched_count=2, acknowledged=True)
mock_mongo_collection.delete_one.return_value = MagicMock(deleted_count=1, acknowledged=True)
mock_mongo_collection.delete_many.return_value = MagicMock(deleted_count=2, acknowledged=True)
mock_mongo_collection.count_documents.return_value = 0
# Start the patch
with patch("pymongo.MongoClient", return_value=mock_mongo_client):
yield mock_mongo_client
@pytest.fixture
def mock_celery():
"""
apps/api/plane/tests/contract/app/test_api_token.py
+17
View File
@@ -366,6 +366,23 @@ class TestApiTokenEndpoint:
create_api_token_for_user.refresh_from_db()
assert create_api_token_for_user.user_type == 0
@pytest.mark.django_db
def test_patch_cannot_modify_allowed_rate_limit(self, session_client, create_user, create_api_token_for_user):
"""Test that allowed_rate_limit cannot be modified via PATCH"""
# Arrange
session_client.force_authenticate(user=create_user)
url = reverse("api-tokens-details", kwargs={"pk": create_api_token_for_user.pk})
original_rate_limit = create_api_token_for_user.allowed_rate_limit
update_data = {"allowed_rate_limit": "100000/min"}
# Act
response = session_client.patch(url, update_data, format="json")
# Assert
assert response.status_code == status.HTTP_200_OK
create_api_token_for_user.refresh_from_db()
assert create_api_token_for_user.allowed_rate_limit == original_rate_limit
@pytest.mark.django_db
def test_patch_cannot_modify_service_token(self, session_client, create_user):
"""Test that service tokens cannot be modified through user token endpoint"""
apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py
+143
View File
@@ -0,0 +1,143 @@
# Copyright (c) 2023-present Plane Software, Inc. and contributors
# SPDX-License-Identifier: AGPL-3.0-only
# See the LICENSE file for details.
"""
Unit tests for the log cleanup tasks.
Verifies that API activity logs past the retention window are hard-deleted
(removed from PostgreSQL, not soft-deleted) and that fresh logs are retained.
"""
from datetime import timedelta
import pytest
from django.conf import settings
from django.utils import timezone
from uuid import uuid4
from plane.bgtasks.cleanup_task import (
delete_api_logs,
delete_email_notification_logs,
delete_webhook_logs,
process_cleanup_task,
)
from plane.db.models import APIActivityLog, EmailNotificationLog, WebhookLog
from plane.tests.factories import UserFactory, WorkspaceFactory
def _make_api_log(created_at):
log = APIActivityLog.objects.create(
token_identifier="hashed-token",
path="/api/v1/workspaces/",
method="GET",
response_code=200,
)
# created_at is auto-set on insert, so backdate it explicitly afterwards.
APIActivityLog.all_objects.filter(pk=log.pk).update(created_at=created_at)
return log
def _make_webhook_log(workspace, created_at):
log = WebhookLog.objects.create(
workspace=workspace,
webhook=uuid4(),
event_type="issue",
request_method="POST",
response_status="200",
)
WebhookLog.all_objects.filter(pk=log.pk).update(created_at=created_at)
return log
def _make_email_log(user, sent_at):
return EmailNotificationLog.objects.create(
receiver=user,
triggered_by=user,
entity_name="issue",
entity="issue",
sent_at=sent_at,
)
@pytest.mark.unit
@pytest.mark.django_db
class TestDeleteApiLogs:
def test_expired_logs_are_hard_deleted(self):
retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS
expired = _make_api_log(timezone.now() - timedelta(days=retention_days + 1))
delete_api_logs()
# Hard delete: the row must be gone even from the unfiltered manager.
assert not APIActivityLog.all_objects.filter(pk=expired.pk).exists()
def test_recent_logs_are_retained(self):
retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS
recent = _make_api_log(timezone.now() - timedelta(days=retention_days - 1))
delete_api_logs()
assert APIActivityLog.all_objects.filter(pk=recent.pk).exists()
@pytest.mark.unit
@pytest.mark.django_db
class TestDeleteWebhookLogs:
def test_expired_logs_are_hard_deleted(self):
workspace = WorkspaceFactory()
retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS
expired = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days + 1))
delete_webhook_logs()
assert not WebhookLog.all_objects.filter(pk=expired.pk).exists()
def test_recent_logs_are_retained(self):
workspace = WorkspaceFactory()
retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS
recent = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days - 1))
delete_webhook_logs()
assert WebhookLog.all_objects.filter(pk=recent.pk).exists()
@pytest.mark.unit
@pytest.mark.django_db
class TestDeleteEmailLogs:
def test_expired_logs_are_hard_deleted(self):
user = UserFactory()
retention_days = settings.EMAIL_LOG_RETENTION_DAYS
expired = _make_email_log(user, timezone.now() - timedelta(days=retention_days + 1))
delete_email_notification_logs()
assert not EmailNotificationLog.all_objects.filter(pk=expired.pk).exists()
def test_recent_logs_are_retained(self):
user = UserFactory()
retention_days = settings.EMAIL_LOG_RETENTION_DAYS
recent = _make_email_log(user, timezone.now() - timedelta(days=retention_days - 1))
delete_email_notification_logs()
assert EmailNotificationLog.all_objects.filter(pk=recent.pk).exists()
@pytest.mark.unit
class TestProcessCleanupTaskErrorHandling:
def test_batch_delete_failure_is_swallowed(self):
"""A failing batch is logged and skipped; the run does not raise."""
class _BoomManager:
@staticmethod
def filter(**kwargs):
raise RuntimeError("db unavailable")
class _BoomModel:
all_objects = _BoomManager()
# Should not raise even though the delete blows up.
process_cleanup_task(lambda: iter([1, 2, 3]), _BoomModel, "Boom")
apps/api/plane/tests/unit/middleware/test_logger.py
+79
View File
@@ -0,0 +1,79 @@
# Copyright (c) 2023-present Plane Software, Inc. and contributors
# SPDX-License-Identifier: AGPL-3.0-only
# See the LICENSE file for details.
"""
Unit tests for APITokenLogMiddleware.
Covers the credential-hygiene guarantees of the external API request logger:
- the raw API key is never persisted (a non-reversible hash is stored instead)
- sensitive request headers are redacted before being logged
"""
import hashlib
import hmac
from unittest.mock import Mock, patch
import pytest
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.http import HttpResponse
from django.test import RequestFactory
from plane.middleware.logger import APITokenLogMiddleware
@pytest.fixture
def request_factory():
return RequestFactory()
@pytest.fixture
def middleware():
return APITokenLogMiddleware(Mock(return_value=HttpResponse(b"{}")))
@pytest.mark.unit
class TestAPITokenLogMiddleware:
API_KEY = "plane_api_supersecretvalue"
AUTHORIZATION = "Bearer secret-bearer-token"
COOKIE = "sessionid=secret-session-value"
def _captured_log_data(self, middleware, request_factory):
request = request_factory.get(
"/api/v1/workspaces/",
HTTP_X_API_KEY=self.API_KEY,
HTTP_AUTHORIZATION=self.AUTHORIZATION,
HTTP_COOKIE=self.COOKIE,
)
request.user = AnonymousUser()
response = HttpResponse(b"{}")
with patch("plane.middleware.logger.process_logs") as process_logs:
middleware.process_request(request, response, request_body=b"")
assert process_logs.delay.called
return process_logs.delay.call_args.kwargs["log_data"]
def test_token_identifier_is_hashed_not_plaintext(self, middleware, request_factory):
log_data = self._captured_log_data(middleware, request_factory)
expected_hash = hmac.new(
settings.SECRET_KEY.encode(), self.API_KEY.encode(), hashlib.sha256
).hexdigest()
assert log_data["token_identifier"] == expected_hash
assert self.API_KEY not in log_data["token_identifier"]
def test_sensitive_headers_are_redacted(self, middleware, request_factory):
log_data = self._captured_log_data(middleware, request_factory)
# None of the sensitive header values may appear in the logged headers.
assert self.API_KEY not in log_data["headers"]
assert self.AUTHORIZATION not in log_data["headers"]
assert self.COOKIE not in log_data["headers"]
assert "[REDACTED]" in log_data["headers"]
def test_no_log_without_api_key(self, middleware, request_factory):
request = request_factory.get("/api/v1/workspaces/")
request.user = AnonymousUser()
with patch("plane.middleware.logger.process_logs") as process_logs:
middleware.process_request(request, HttpResponse(b"{}"), request_body=b"")
assert not process_logs.delay.called
apps/api/plane/tests/unit/settings/test_retention.py
+34
View File
@@ -0,0 +1,34 @@
# Copyright (c) 2023-present Plane Software, Inc. and contributors
# SPDX-License-Identifier: AGPL-3.0-only
# See the LICENSE file for details.
"""Unit tests for the log-retention env parsing helper."""
import pytest
from plane.settings.common import _retention_days
ENV_VAR = "TEST_RETENTION_DAYS"
@pytest.mark.unit
class TestRetentionDays:
def test_uses_default_when_unset(self, monkeypatch):
monkeypatch.delenv(ENV_VAR, raising=False)
assert _retention_days(ENV_VAR, 14) == 14
def test_uses_env_value_when_valid(self, monkeypatch):
monkeypatch.setenv(ENV_VAR, "30")
assert _retention_days(ENV_VAR, 14) == 30
def test_zero_is_allowed(self, monkeypatch):
monkeypatch.setenv(ENV_VAR, "0")
assert _retention_days(ENV_VAR, 14) == 0
def test_negative_falls_back_to_default(self, monkeypatch):
monkeypatch.setenv(ENV_VAR, "-5")
assert _retention_days(ENV_VAR, 14) == 14
def test_unparseable_falls_back_to_default(self, monkeypatch):
monkeypatch.setenv(ENV_VAR, "abc")
assert _retention_days(ENV_VAR, 7) == 7
apps/api/requirements/base.txt
-2
View File
@@ -9,8 +9,6 @@ psycopg==3.3.0
psycopg-binary==3.3.0
psycopg-c==3.3.0
dj-database-url==2.1.0
# mongo
pymongo==4.6.3
# redis
redis==5.0.4
django-redis==5.4.0