I’m running a GitLab container in docker. When I’m trying to save settings in Admin Area (in specific, Settings – General), it hangs for 60 sec and redirect me to 500 page.
The server run in an air-gapped intranet, without Internet connection.
Here’s the docker-compose.yml file I’m using:
version: '3.9'
services:
gitlab:
image: 'gitlab/gitlab-ce:16.6.2-ce.0'
ports:
- '20080:80'
- '20443:443'
- '20022:22'
shm_size: '1gb'
The possible related log is shown below:
{
"method": "PATCH",
"path": "/admin/application_settings/general",
"format": "html",
"controller": "Admin::ApplicationSettingsController",
"action": "general",
"status": 500,
"time": "2023-12-18T07:59:15.696Z",
"params": [
{
"key": "_method",
"value": "patch"
},
{
"key": "authenticity_token",
"value": "[FILTERED]"
},
{
"key": "application_setting",
"value": {
"import_sources": [
"",
"gitlab_project"
],
"project_export_enabled": "1",
"bulk_import_enabled": "0",
"max_export_size": "0",
"max_import_size": "0",
"max_import_remote_file_size": "10240",
"bulk_import_max_download_file_size": "5120",
"max_decompressed_archive_size": "25600",
"decompress_archive_file_timeout": "210"
}
}
],
"correlation_id": "01HHXYXH7MKN0EKV8DSJG3KCP0",
"meta.caller_id": "Admin::ApplicationSettingsController#general",
"meta.remote_ip": "12.70.170.3",
"meta.feature_category": "not_owned",
"meta.user": "root",
"meta.user_id": 1,
"meta.client_id": "user/1",
"remote_ip": "12.70.170.3",
"user_id": 1,
"username": "root",
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
"queue_duration_s": 0.061149,
"request_urgency": "default",
"target_duration_s": 1,
"redis_calls": 12,
"redis_allowed_cross_slot_calls": 1,
"redis_duration_s": 0.006305,
"redis_read_bytes": 1492,
"redis_write_bytes": 1127,
"redis_feature_flag_calls": 7,
"redis_feature_flag_duration_s": 0.003357,
"redis_feature_flag_read_bytes": 1312,
"redis_feature_flag_write_bytes": 436,
"redis_sessions_calls": 3,
"redis_sessions_allowed_cross_slot_calls": 1,
"redis_sessions_duration_s": 0.002376,
"redis_sessions_read_bytes": 180,
"redis_sessions_write_bytes": 587,
"redis_shared_state_calls": 2,
"redis_shared_state_duration_s": 0.000572,
"redis_shared_state_write_bytes": 104,
"db_count": 4,
"db_write_count": 0,
"db_cached_count": 1,
"db_replica_count": 0,
"db_primary_count": 4,
"db_main_count": 4,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 1,
"db_main_cached_count": 1,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.0,
"db_primary_duration_s": 0.013,
"db_main_duration_s": 0.013,
"db_ci_duration_s": 0.0,
"db_main_replica_duration_s": 0.0,
"db_ci_replica_duration_s": 0.0,
"cpu_s": 0.389744,
"mem_objects": 183156,
"mem_bytes": 19300163,
"mem_mallocs": 84754,
"mem_total_bytes": 26626403,
"pid": 1108,
"worker_id": "puma_2",
"rate_limiting_gates": [],
"exception.class": "ActiveRecord::StatementInvalid",
"exception.message": "PG::ConnectionBad: PQconsumeInput() FATAL: terminating connection due to idle-in-transaction timeout\nserver closed the connection unexpectedly\n\tThis probably means the server terminated abnormally\n\tbefore or while processing the request.\n",
"exception.backtrace": [
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:137:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:224:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:126:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"app/services/application_settings/update_service.rb:51:in `update_settings'",
"app/services/application_settings/update_service.rb:12:in `execute'",
"app/controllers/admin/application_settings_controller.rb:210:in `perform_update'",
"app/controllers/admin/application_settings_controller.rb:50:in `block (2 levels) in <class:ApplicationSettingsController>'",
"app/controllers/application_controller.rb:498:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:489:in `set_session_storage'",
"lib/gitlab/i18n.rb:114:in `with_locale'",
"lib/gitlab/i18n.rb:120:in `with_user_locale'",
"app/controllers/application_controller.rb:480:in `set_locale'",
"app/controllers/application_controller.rb:473:in `set_current_context'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/path_traversal_check.rb:48:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:44:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"exception.cause_class": "PG::ConnectionBad",
"exception.sql": "/*application:web,correlation_id:01HHXYXH7MKN0EKV8DSJG3KCP0,endpoint_id:Admin::ApplicationSettingsController#general,db_config_name:main*/ ROLLBACK",
"db_duration_s": 0.43426,
"view_duration_s": 0.0,
"duration_s": 60.6218
}
{
"severity": "ERROR",
"time": "2023-12-15T07:04:14.029Z",
"correlation_id": "01HHP4KVN86B6FY4BQP52C3HD1",
"meta.caller_id": "ExternalServiceReactiveCachingWorker",
"meta.remote_ip": "12.70.170.3",
"meta.feature_category": "groups_and_projects",
"meta.user": "root",
"meta.user_id": 1,
"meta.client_id": "user/1",
"meta.related_class": "VersionCheck",
"meta.root_caller_id": "RootController#index",
"exception.class": "Gitlab::HTTP_V2::BlockedUrlError",
"exception.message": "URL is blocked: Host cannot be resolved or invalid",
"exception.backtrace": [
"lib/gitlab/http_connection_adapter.rb:60:in `rescue in validate_url_with_proxy!'",
"lib/gitlab/http_connection_adapter.rb:52:in `validate_url_with_proxy!'",
"lib/gitlab/http_connection_adapter.rb:29:in `connection'",
"lib/gitlab/legacy_http.rb:37:in `perform_request'",
"lib/gitlab/http.rb:48:in `public_send'",
"lib/gitlab/http.rb:48:in `block (2 levels) in singleton class'",
"lib/gitlab/http.rb:54:in `try_get'",
"lib/version_check.rb:71:in `calculate_reactive_cache'",
"app/models/concerns/reactive_caching.rb:94:in `block (2 levels) in exclusively_update_reactive_cache!'",
"app/models/concerns/reactive_caching.rb:153:in `enqueuing_update'",
"app/models/concerns/reactive_caching.rb:93:in `block in exclusively_update_reactive_cache!'",
"app/models/concerns/reactive_caching.rb:143:in `locking_reactive_cache'",
"app/models/concerns/reactive_caching.rb:89:in `exclusively_update_reactive_cache!'",
"app/workers/concerns/reactive_cacheable_worker.rb:35:in `perform'",
"lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:29:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:124:in `block in use'",
"lib/gitlab/application_context.rb:124:in `use'",
"lib/gitlab/application_context.rb:62:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:94:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:122:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:122:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:93:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
],
"exception.cause_class": "Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError",
"user.username": "root",
"tags.program": "sidekiq",
"tags.locale": "en",
"tags.feature_category": "groups_and_projects",
"tags.correlation_id": "01HHP4KVN86B6FY4BQP52C3HD1",
"extra.sidekiq": {
"retry": 3,
"queue": "default",
"version": 0,
"args": [
"VersionCheck",
"[FILTERED]"
],
"class": "ExternalServiceReactiveCachingWorker",
"jid": "e6244186fac178cc1209a3c7",
"created_at": 1702623833.9807603,
"correlation_id": "01HHP4KVN86B6FY4BQP52C3HD1",
"meta.caller_id": "RootController#index",
"meta.remote_ip": "12.70.170.3",
"meta.feature_category": "groups_and_projects",
"meta.user": "root",
"meta.user_id": 1,
"meta.client_id": "user/1",
"meta.related_class": "VersionCheck",
"meta.root_caller_id": "RootController#index",
"worker_data_consistency": "always",
"idempotency_key": "resque:gitlab:duplicate:default:b610d442fdc82b690dca61a846f215244f166bae201a51f5e520beb3a0cfe345",
"size_limiter": "validated",
"enqueued_at": 1702623833.9836192
}
}
Related Postgre log /var/log/gitlab/postgresql/current:
FATAL: terminating connection due to idle-in-transaction timeout
Possibly related to an open issue #422312 Gitlab 16.x on Ky10.x86_64 cannt save changes in settings. (Error 500.)
I ran the same configuration on a test server with Internet connection and latest image: 'gitlab/gitlab-ce:latest and it works fine.
Also tried older image: 'gitlab/gitlab-ce:16.5.0-ce.0' whereas the issue remains.