From a9ee0c56dfb11528690146ce717c01dbe614d3ec Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Mon, 13 Oct 2025 15:42:20 +0300 Subject: [PATCH 1/2] Add management command to manual archive --- admin/nodes/views.py | 9 ++ .../process_manual_restart_approvals.py | 145 ++++++++++++++++++ osf/models/admin_log_entry.py | 2 + scripts/check_manual_restart_approval.py | 74 +++++++++ scripts/enhanced_stuck_registration_audit.py | 137 +++++++++++++++++ website/archiver/tasks.py | 21 +++ website/settings/defaults.py | 2 + 7 files changed, 390 insertions(+) create mode 100644 osf/management/commands/process_manual_restart_approvals.py create mode 100644 scripts/check_manual_restart_approval.py create mode 100644 scripts/enhanced_stuck_registration_audit.py diff --git a/admin/nodes/views.py b/admin/nodes/views.py index 74321c8f908..1d4fe36b702 100644 --- a/admin/nodes/views.py +++ b/admin/nodes/views.py @@ -754,6 +754,7 @@ class ForceArchiveRegistrationsView(NodeMixin, View): def post(self, request, *args, **kwargs): # Prevents circular imports that cause admin app to hang at startup from osf.management.commands.force_archive import verify, archive, DEFAULT_PERMISSIBLE_ADDONS + from osf.models.admin_log_entry import update_admin_log, MANUAL_ARCHIVE_RESTART registration = self.get_object() force_archive_params = request.POST @@ -779,6 +780,14 @@ def post(self, request, *args, **kwargs): messages.success(request, f"Registration {registration._id} can be archived.") else: try: + update_admin_log( + user_id=request.user.id, + object_id=registration.pk, + object_repr=str(registration), + message=f'Manual archive restart initiated for registration {registration._id}', + action_flag=MANUAL_ARCHIVE_RESTART + ) + archive( registration, permissible_addons=addons, diff --git a/osf/management/commands/process_manual_restart_approvals.py b/osf/management/commands/process_manual_restart_approvals.py new file mode 100644 index 00000000000..e708320357f --- /dev/null +++ b/osf/management/commands/process_manual_restart_approvals.py @@ -0,0 +1,145 @@ +import logging +from datetime import timedelta +from django.core.management.base import BaseCommand +from django.utils import timezone +from osf.models import Registration +from osf.models.admin_log_entry import AdminLogEntry, MANUAL_ARCHIVE_RESTART +from website import settings +from scripts.approve_registrations import approve_past_pendings + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Process registrations that were manually restarted and may need approval' + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be done without actually doing it', + ) + parser.add_argument( + '--hours-back', + type=int, + default=72, + help='How many hours back to look for manual restarts (default: 72)', + ) + parser.add_argument( + '--registration-id', + type=str, + help='Process a specific registration ID only', + ) + + def handle(self, *args, **options): + dry_run = options['dry_run'] + hours_back = options['hours_back'] + specific_registration = options.get('registration_id') + + if dry_run: + self.stdout.write(self.style.WARNING('Running in DRY RUN mode - no changes will be made')) + + since = timezone.now() - timedelta(hours=hours_back) + + query = AdminLogEntry.objects.filter( + action_flag=MANUAL_ARCHIVE_RESTART, + action_time__gte=since + ) + + if specific_registration: + try: + reg = Registration.objects.get(_id=specific_registration) + query = query.filter(object_id=reg.pk) + self.stdout.write(f"Processing specific registration: {specific_registration}") + except Registration.DoesNotExist: + self.stdout.write(self.style.ERROR(f"Registration {specific_registration} not found")) + return + + manual_restart_logs = query.values_list('object_id', flat=True).distinct() + + registrations_to_check = Registration.objects.filter( + pk__in=manual_restart_logs, + ) + + self.stdout.write(f"Found {registrations_to_check.count()} manually restarted registrations to check") + + approvals_ready = [] + skipped_registrations = [] + + for registration in registrations_to_check: + status = self.should_auto_approve(registration) + + if status == 'ready': + approval = registration.registration_approval + if approval: + approvals_ready.append(approval) + self.stdout.write( + self.style.SUCCESS(f"✓ Queuing registration {registration._id} for approval") + ) + else: + skipped_registrations.append((registration._id, status)) + self.stdout.write( + self.style.WARNING(f"⚠ Skipping registration {registration._id}: {status}") + ) + + if approvals_ready: + if dry_run: + self.stdout.write( + self.style.WARNING(f"DRY RUN: Would approve {len(approvals_ready)} registrations") + ) + else: + try: + approve_past_pendings(approvals_ready, dry_run=False) + self.stdout.write( + self.style.SUCCESS(f"✓ Successfully approved {len(approvals_ready)} manually restarted registrations") + ) + except Exception as e: + self.stdout.write( + self.style.ERROR(f"✗ Error approving registrations: {e}") + ) + else: + self.stdout.write('No registrations ready for approval') + + self.stdout.write(f"Total checked: {registrations_to_check.count()}") + self.stdout.write(f"Ready for approval: {len(approvals_ready)}") + self.stdout.write(f"Skipped: {len(skipped_registrations)}") + + if skipped_registrations: + self.stdout.write('\nSkipped registrations:') + for reg_id, reason in skipped_registrations: + self.stdout.write(f" - {reg_id}: {reason}") + + def should_auto_approve(self, registration): + if registration.is_public: + return 'already public' + + if registration.is_registration_approved: + return 'already approved' + + if registration.archiving: + return 'still archiving' + + archive_job = registration.archive_job + if archive_job and hasattr(archive_job, 'status'): + if archive_job.status not in ['SUCCESS', None]: + return f'archive status: {archive_job.status}' + + approval = registration.registration_approval + if not approval: + return 'no approval object' + + if approval.is_approved: + return 'approval already approved' + + if approval.is_rejected: + return 'approval was rejected' + + time_since_initiation = timezone.now() - approval.initiation_date + if time_since_initiation < settings.REGISTRATION_APPROVAL_TIME: + remaining = settings.REGISTRATION_APPROVAL_TIME - time_since_initiation + return f'not ready yet ({remaining} remaining)' + + if registration.is_stuck_registration: + return 'registration still stuck' + + return 'ready' diff --git a/osf/models/admin_log_entry.py b/osf/models/admin_log_entry.py index 169bbe3faef..5c6df117f3a 100644 --- a/osf/models/admin_log_entry.py +++ b/osf/models/admin_log_entry.py @@ -34,6 +34,8 @@ PREPRINT_REMOVED = 70 PREPRINT_RESTORED = 71 +MANUAL_ARCHIVE_RESTART = 80 + def update_admin_log(user_id, object_id, object_repr, message, action_flag=UNKNOWN): AdminLogEntry.objects.log_action( user_id=user_id, diff --git a/scripts/check_manual_restart_approval.py b/scripts/check_manual_restart_approval.py new file mode 100644 index 00000000000..e9e6c70de0c --- /dev/null +++ b/scripts/check_manual_restart_approval.py @@ -0,0 +1,74 @@ +import logging +from framework.celery_tasks import app as celery_app +from django.core.management import call_command +from osf.models import Registration + +logger = logging.getLogger(__name__) + + +@celery_app.task(name='scripts.check_manual_restart_approval') +def check_manual_restart_approval(registration_id): + try: + try: + registration = Registration.objects.get(_id=registration_id) + except Registration.DoesNotExist: + logger.error(f"Registration {registration_id} not found") + return f"Registration {registration_id} not found" + + if registration.is_public or registration.is_registration_approved: + return f"Registration {registration_id} already approved/public" + + if registration.archiving: + logger.info(f"Registration {registration_id} still archiving, retrying in 10 minutes") + check_manual_restart_approval.apply_async( + args=[registration_id], + countdown=600 + ) + return f"Registration {registration_id} still archiving, scheduled retry" + + logger.info(f"Processing manual restart approval for registration {registration_id}") + + call_command( + 'process_manual_restart_approvals', + registration_id=registration_id, + dry_run=False, + hours_back=24, + verbosity=1 + ) + + return f"Processed manual restart approval check for registration {registration_id}" + + except Exception as e: + logger.error(f"Error processing manual restart approval for {registration_id}: {e}") + raise + + +@celery_app.task(name='scripts.check_manual_restart_approvals_batch') +def check_manual_restart_approvals_batch(hours_back=24): + try: + logger.info(f"Running batch check for manual restart approvals (last {hours_back} hours)") + + call_command( + 'process_manual_restart_approvals', + dry_run=False, + hours_back=hours_back, + verbosity=1 + ) + + return f"Completed batch manual restart approval check for last {hours_back} hours" + + except Exception as e: + logger.error(f"Error in batch manual restart approval check: {e}") + raise + + +@celery_app.task(name='scripts.delayed_manual_restart_approval') +def delayed_manual_restart_approval(registration_id, delay_minutes=30): + logger.info(f"Scheduling delayed manual restart approval check for {registration_id} in {delay_minutes} minutes") + + check_manual_restart_approval.apply_async( + args=[registration_id], + countdown=delay_minutes * 60 + ) + + return f"Scheduled manual restart approval check for {registration_id} in {delay_minutes} minutes" \ No newline at end of file diff --git a/scripts/enhanced_stuck_registration_audit.py b/scripts/enhanced_stuck_registration_audit.py new file mode 100644 index 00000000000..3905526ae52 --- /dev/null +++ b/scripts/enhanced_stuck_registration_audit.py @@ -0,0 +1,137 @@ +import logging + +from django.core.management import call_command +from framework.celery_tasks import app as celery_app +from osf.models import Registration +from osf.management.commands.force_archive import archive, DEFAULT_PERMISSIBLE_ADDONS +from scripts.stuck_registration_audit import analyze_failed_registration_nodes + +logger = logging.getLogger(__name__) + + +@celery_app.task(name='scripts.enhanced_stuck_registration_audit') +def enhanced_stuck_registration_audit(): + logger.info('Starting enhanced stuck registration audit') + + try: + logger.info('Processing pending manual restart approvals') + call_command('process_manual_restart_approvals', dry_run=False, hours_back=72) + except Exception as e: + logger.error(f"Error processing manual restart approvals: {e}") + + logger.info('Analyzing failed registrations') + failed_registrations = analyze_failed_registration_nodes() + + if not failed_registrations: + logger.info('No failed registrations found') + return 'No failed registrations found' + + logger.info(f"Found {len(failed_registrations)} failed registrations") + + auto_retryable = [] + needs_manual_intervention = [] + + for reg_info in failed_registrations: + registration_id = reg_info['registration'] + + try: + registration = Registration.objects.get(_id=registration_id) + + if should_auto_retry(reg_info, registration): + auto_retryable.append(registration) + logger.info(f"Registration {registration_id} eligible for auto-retry") + else: + needs_manual_intervention.append(reg_info) + logger.info(f"Registration {registration_id} needs manual intervention") + + except Registration.DoesNotExist: + logger.warning(f"Registration {registration_id} not found") + needs_manual_intervention.append(reg_info) + continue + + successfully_retried = [] + failed_auto_retries = [] + + for reg in auto_retryable: + try: + logger.info(f"Attempting auto-retry for stuck registration {reg._id}") + + archive( + reg, + permissible_addons=DEFAULT_PERMISSIBLE_ADDONS, + allow_unconfigured=True, + skip_collisions=True + ) + + successfully_retried.append(reg._id) + logger.info(f"Successfully auto-retried registration {reg._id}") + + except Exception as e: + logger.error(f"Auto-retry failed for registration {reg._id}: {e}") + failed_auto_retries.append({ + 'registration': reg._id, + 'auto_retry_error': str(e), + 'original_info': next(info for info in failed_registrations if info['registration'] == reg._id) + }) + + needs_manual_intervention.extend(failed_auto_retries) + + logger.info(f"Auto-retry results: {len(successfully_retried)} successful, {len(failed_auto_retries)} failed") + + summary = { + 'total_failed': len(failed_registrations), + 'auto_retried_success': len(successfully_retried), + 'auto_retried_failed': len(failed_auto_retries), + 'needs_manual': len(needs_manual_intervention), + 'successfully_retried_ids': successfully_retried + } + + logger.info(f"Enhanced audit completed: {summary}") + return summary + + +def should_auto_retry(reg_info, registration): + if not reg_info.get('can_be_reset', False): + return False + + addon_list = reg_info.get('addon_list', []) + complex_addons = set(addon_list) - {'osfstorage', 'wiki'} + if complex_addons: + logger.info(f"Registration {registration._id} has complex addons: {complex_addons}") + return False + + logs_after_reg = reg_info.get('logs_on_original_after_registration_date', []) + if logs_after_reg: + logger.info(f"Registration {registration._id} has post-registration logs: {logs_after_reg}") + return False + + successful_after = reg_info.get('succeeded_registrations_after_failed', []) + if successful_after: + logger.info(f"Registration {registration._id} has successful registrations after failure: {successful_after}") + return False + + import django.utils.timezone as timezone + from datetime import timedelta + if registration.registered_date: + age = timezone.now() - registration.registered_date + if age > timedelta(days=30): + logger.info(f"Registration {registration._id} is too old ({age.days} days)") + return False + return True + +@celery_app.task(name='scripts.manual_restart_approval_batch') +def manual_restart_approval_batch(): + logger.info('Running manual restart approval batch task') + + try: + from scripts.check_manual_restart_approval import check_manual_restart_approvals_batch + result = check_manual_restart_approvals_batch.delay(hours_back=24) + return f"Queued manual restart approval batch task: {result.id}" + except Exception as e: + logger.error(f"Error running manual restart approval batch: {e}") + raise + + +if __name__ == '__main__': + result = enhanced_stuck_registration_audit() + print(f"Audit completed: {result}") \ No newline at end of file diff --git a/website/archiver/tasks.py b/website/archiver/tasks.py index 42e5bfb568b..9adecc620da 100644 --- a/website/archiver/tasks.py +++ b/website/archiver/tasks.py @@ -7,6 +7,9 @@ import celery from celery.utils.log import get_task_logger +from django.utils import timezone +from datetime import timedelta + from framework.celery_tasks import app as celery_app from framework.celery_tasks.utils import logged from framework.exceptions import HTTPError @@ -29,9 +32,13 @@ from website.archiver.utils import normalize_unicode_filenames from website.archiver import signals as archiver_signals +from scripts.check_manual_restart_approval import delayed_manual_restart_approval + from website.project import signals as project_signals from website import settings from website.app import init_addons + +from osf.models.admin_log_entry import AdminLogEntry, MANUAL_ARCHIVE_RESTART from osf.models import ( ArchiveJob, AbstractNode, @@ -369,4 +376,18 @@ def archive_success(self, dst_pk, job_pk): job.save() dst.sanction.ask(dst.get_active_contributors_recursive(unique_users=True)) + if was_manually_restarted(dst): + logger.info(f'Registration {dst._id} was manually restarted, scheduling approval check') + delayed_manual_restart_approval.delay(dst._id, delay_minutes=5) + dst.update_search() + + +def was_manually_restarted(registration): + recent_logs = AdminLogEntry.objects.filter( + object_id=registration.pk, + action_flag=MANUAL_ARCHIVE_RESTART, + action_time__gte=timezone.now() - timedelta(hours=48) + ) + + return recent_logs.exists() diff --git a/website/settings/defaults.py b/website/settings/defaults.py index aaa8173acf9..c727f32dab2 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -476,6 +476,8 @@ class CeleryConfig: 'scripts.add_missing_identifiers_to_preprints', 'osf.management.commands.approve_pending_schema_response', 'api.share.utils', + 'scripts.check_manual_restart_approval', + 'scripts.enhanced_stuck_registration_audit', } try: From 3af6f4bd9a7d965ff0199b709c0679b5a695e44d Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Fri, 24 Oct 2025 13:42:10 +0300 Subject: [PATCH 2/2] fix test --- api/institutions/serializers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/institutions/serializers.py b/api/institutions/serializers.py index fac33923905..6f4bc4f9e15 100644 --- a/api/institutions/serializers.py +++ b/api/institutions/serializers.py @@ -230,6 +230,11 @@ class Meta: name = ser.CharField(read_only=True) number_of_users = ser.IntegerField(read_only=True) + def get_absolute_url(self, obj): + institution_id = self.context['request'].parser_context['kwargs']['institution_id'] + dept_id = obj['name'].replace(' ', '-') + return f'/institutions/{institution_id}/metrics/departments/{dept_id}/' + class InstitutionUserMetricsSerializer(JSONAPISerializer): '''serializer for institution-users metrics @@ -285,6 +290,10 @@ def get_contacts(self, obj): ).order_by('sender_name') return list(results) + def get_absolute_url(self, obj): + institution_id = self.context['request'].parser_context['kwargs']['institution_id'] + return f'/institutions/{institution_id}/metrics/users/' + class InstitutionSummaryMetricsSerializer(JSONAPISerializer): '''serializer for institution-summary metrics @@ -316,6 +325,10 @@ class Meta: related_view_kwargs={'institution_id': ''}, ) + def get_absolute_url(self, obj): + institution_id = self.context['request'].parser_context['kwargs']['institution_id'] + return f'/institutions/{institution_id}/metrics/summary/' + class InstitutionRelated(JSONAPIRelationshipSerializer): id = ser.CharField(source='_id', required=False, allow_null=True)