Course structure not being found in the database

Our tutor openedx platform was installed from koa version. We successfully upgraded it into opened tutor nutmeg. But after the errors in image build, we are forced to upgrade it into the latest versions.

After successfully upgrading our nutmeg version into palm 16.1.6, one of our courses encountered an error in studio.

Here are the traceback logs:

cms-1  | Traceback (most recent call last):
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 188, in bulk_operations
cms-1  |     yield
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 1042, in bulk_operations
cms-1  |     yield
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 713, in course_index
cms-1  |     course_structure = _course_outline_json(request, course_block)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 379, in _course_outline_json
cms-1  |     return create_xblock_info(
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/edx_django_utils/plugins/pluggable_override.py", line 77, in wrapper
cms-1  |     return prev_fn(*args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/block.py", line 1172, in create_xblock_info
cms-1  |     has_changes = modulestore().has_changes(xblock)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 963, in has_changes
cms-1  |     return store.has_changes(xblock)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split_draft.py", line 346, in has_changes
cms-1  |     published_course = get_course(ModuleStoreEnum.BranchName.published)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split_draft.py", line 340, in get_course
cms-1  |     return self._lookup_course(xblock.location.course_key.for_branch(branch_name)).structure
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split.py", line 860, in _lookup_course
cms-1  |     raise ItemNotFoundError(f'Structure: {version_guid}')
cms-1  | xmodule.modulestore.exceptions.ItemNotFoundError: Structure: 661e1fc34ff1cc812cbaaea0
cms-1  |
cms-1  | During handling of the above exception, another exception occurred:
cms-1  |
cms-1  | Traceback (most recent call last):
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47, in inner
cms-1  |     response = get_response(request)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 181, in _get_response
cms-1  |     response = wrapped_callback(request, *callback_args, **callback_kwargs)
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 75, in inner
cms-1  |     return func(*args, **kwds)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
cms-1  |     return view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 310, in course_handler
cms-1  |     return course_index(request, CourseKey.from_string(course_key_string))
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
cms-1  |     return view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/utils/decorators.py", line 130, in _wrapped_view
cms-1  |     response = view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 741, in course_index
cms-1  |     return render_to_response('course_outline.html', {
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 131, in __exit__
cms-1  |     self.gen.throw(type, value, traceback)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 1042, in bulk_operations
cms-1  |     yield
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 131, in __exit__
cms-1  |     self.gen.throw(type, value, traceback)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 190, in bulk_operations
cms-1  |     self._end_bulk_operation(course_id, emit_signals, ignore_case)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 286, in _end_bulk_operation
cms-1  |     dirty = self._end_outermost_bulk_operation(bulk_ops_record, structure_key)  # lint-amnesty, pylint: disable=assignment-from-no-return
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split.py", line 263, in _end_outermost_bulk_operation
cms-1  |     self.db_connection.insert_structure(bulk_write_record.structures[_id], bulk_write_record.course_key)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/mongo_connection.py", line 368, in insert_structure
cms-1  |     tagger.measure("blocks", len(structure["blocks"]))
cms-1  | TypeError: 'NoneType' object is not subscriptable
cms-1  | 2024-04-18 06:51:27,291 ERROR 17 [django.request] [user 149781] [ip 162.158.222.244] log.py:224 - Internal Server Error: /course/course-v1:Coursebank+FSA-DM004+2024_Q2
cms-1  | Traceback (most recent call last):
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 188, in bulk_operations
cms-1  |     yield
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 1042, in bulk_operations
cms-1  |     yield
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 713, in course_index
cms-1  |     course_structure = _course_outline_json(request, course_block)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 379, in _course_outline_json
cms-1  |     return create_xblock_info(
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/edx_django_utils/plugins/pluggable_override.py", line 77, in wrapper
cms-1  |     return prev_fn(*args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/block.py", line 1172, in create_xblock_info
cms-1  |     has_changes = modulestore().has_changes(xblock)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 963, in has_changes
cms-1  |     return store.has_changes(xblock)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split_draft.py", line 346, in has_changes
cms-1  |     published_course = get_course(ModuleStoreEnum.BranchName.published)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split_draft.py", line 340, in get_course
cms-1  |     return self._lookup_course(xblock.location.course_key.for_branch(branch_name)).structure
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split.py", line 860, in _lookup_course
cms-1  |     raise ItemNotFoundError(f'Structure: {version_guid}')
cms-1  | xmodule.modulestore.exceptions.ItemNotFoundError: Structure: 661e1fc34ff1cc812cbaaea0
cms-1  |
cms-1  | During handling of the above exception, another exception occurred:
cms-1  |
cms-1  | Traceback (most recent call last):
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47, in inner
cms-1  |     response = get_response(request)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 181, in _get_response
cms-1  |     response = wrapped_callback(request, *callback_args, **callback_kwargs)
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 75, in inner
cms-1  |     return func(*args, **kwds)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
cms-1  |     return view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 310, in course_handler
cms-1  |     return course_index(request, CourseKey.from_string(course_key_string))
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
cms-1  |     return view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/venv/lib/python3.8/site-packages/django/utils/decorators.py", line 130, in _wrapped_view
cms-1  |     response = view_func(request, *args, **kwargs)
cms-1  |   File "/openedx/edx-platform/./cms/djangoapps/contentstore/views/course.py", line 741, in course_index
cms-1  |     return render_to_response('course_outline.html', {
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 131, in __exit__
cms-1  |     self.gen.throw(type, value, traceback)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/mixed.py", line 1042, in bulk_operations
cms-1  |     yield
cms-1  |   File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 131, in __exit__
cms-1  |     self.gen.throw(type, value, traceback)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 190, in bulk_operations
cms-1  |     self._end_bulk_operation(course_id, emit_signals, ignore_case)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/__init__.py", line 286, in _end_bulk_operation
cms-1  |     dirty = self._end_outermost_bulk_operation(bulk_ops_record, structure_key)  # lint-amnesty, pylint: disable=assignment-from-no-return
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/split.py", line 263, in _end_outermost_bulk_operation
cms-1  |     self.db_connection.insert_structure(bulk_write_record.structures[_id], bulk_write_record.course_key)
cms-1  |   File "/openedx/edx-platform/xmodule/modulestore/split_mongo/mongo_connection.py", line 368, in insert_structure
cms-1  |     tagger.measure("blocks", len(structure["blocks"]))
cms-1  | TypeError: 'NoneType' object is not subscriptable