Problems with load balancer, possible proxy module bug

Henry GALVEZ <henry.galvez@xxxxxxx> · Mon, 7 Mar 2022 15:29:49 +0000

Hi to all

We have a host that evaluates access with a Balancer and a HealthCheck.

In version 2.4.43, if the HealthCheck's status is 1 and after a certain time it returns to 0, everything goes back to normal.

But from version 2.4.46 until version 2.4.52, when HealthCheck returns to status 0, it continues to throw the following error: "All Workers Are In Error State" and forces the restart of the HTTPD service.

Configuration example:

```config
#################################################################
## httpd.conf ##
##################################################################

ServerRoot /usr/local/apache2

LoadModule autoindex_module             /usr/local/apache2/modules/mod_autoindex.so
LoadModule unique_id_module             /usr/local/apache2/modules/mod_unique_id.so
#LoadModule mpm_worker_module           /usr/local/apache2/modules/mod_mpm_worker.so
LoadModule authn_core_module            /usr/local/apache2/modules/mod_authn_core.so
LoadModule authz_core_module            /usr/local/apache2/modules/mod_authz_core.so
LoadModule auth_basic_module            /usr/local/apache2/modules/mod_auth_basic.so
LoadModule authn_file_module            /usr/local/apache2/modules/mod_authn_file.so
LoadModule authz_user_module            /usr/local/apache2/modules/mod_authz_user.so
LoadModule mpm_event_module             /usr/local/apache2/modules/mod_mpm_event.so
LoadModule headers_module               /usr/local/apache2/modules/mod_headers.so
LoadModule authz_host_module            /usr/local/apache2/modules/mod_authz_host.so
LoadModule deflate_module               /usr/local/apache2/modules/mod_deflate.so
LoadModule setenvif_module              /usr/local/apache2/modules/mod_setenvif.so
LoadModule userdir_module               /usr/local/apache2/modules/mod_userdir.so
LoadModule dir_module                   /usr/local/apache2/modules/mod_dir.so
LoadModule expires_module               /usr/local/apache2/modules/mod_expires.so
LoadModule rewrite_module               /usr/local/apache2/modules/mod_rewrite.so
LoadModule alias_module                 /usr/local/apache2/modules/mod_alias.so
LoadModule mime_module                  /usr/local/apache2/modules/mod_mime.so
LoadModule status_module                /usr/local/apache2/modules/mod_status.so
LoadModule proxy_module                 /usr/local/apache2/modules/mod_proxy.so
LoadModule proxy_http_module            /usr/local/apache2/modules/mod_proxy_http.so
LoadModule cache_module                 /usr/local/apache2/modules/mod_cache.so
LoadModule proxy_ajp_module             /usr/local/apache2/modules/mod_proxy_ajp.so
LoadModule ssl_module                   /usr/local/apache2/modules/mod_ssl.so
LoadModule proxy_balancer_module        /usr/local/apache2/modules/mod_proxy_balancer.so
LoadModule slotmem_shm_module           /usr/local/apache2/modules/mod_slotmem_shm.so
LoadModule socache_shmcb_module         /usr/local/apache2/modules/mod_socache_shmcb.so
LoadModule lbmethod_byrequests_module   /usr/local/apache2/modules/mod_lbmethod_byrequests.so
LoadModule info_module                  /usr/local/apache2/modules/mod_info.so
LoadModule proxy_hcheck_module          /usr/local/apache2/modules/mod_proxy_hcheck.so
LoadModule unixd_module                 /usr/local/apache2/modules/mod_unixd.so
LoadModule log_config_module            /usr/local/apache2/modules/mod_log_config.so
LoadModule watchdog_module              /usr/local/apache2/modules/mod_watchdog.so

<IfModule proxy_hcheck_module>
    LogLevel proxy_hcheck:trace8 proxy:trace8
</IfModule>

Timeout 1800
KeepAlive on
MaxRequestWorkers 8192
ServerLimit 128
ThreadsPerChild 64
MaxRequestsPerChild 1000
StartServers 3
MinSpareThreads 5
MaxSpareThreads 250
MaxKeepAliveRequests 250
KeepAliveTimeout 5
AsyncRequestWorkerFactor 2

userdir disabled root

TypesConfig /usr/local/apache2/conf/mime.types

AddEncoding x-compress Z
AddEncoding x-gzip gz tgz

############################################
# Definitions
############################################

Listen ######:443

ServerName ######

ServerTokens Prod
DeflateFilterNote ratio

# Serveur HTTPD
DocumentRoot /usr/local/apache2/htdocs/
DirectoryIndex index.html index.htm index.jsp

# Filtering accessLog to delete images, CSS and JavaScript
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla|css|js|pdf|ico)$ dontlog

RequestHeader setifempty Correlation-ID %{UNIQUE_ID}e
SetEnvIf Correlation-ID "(.*)" MyCorrelationId=$1
Header setifempty Correlation-ID "%{MyCorrelationId}e"

# Recovery of the original applicant only (X-Forwarded-For = host1, host2, host3... If we pass through several layers)
RequestHeader setIfEmpty X-Forwarded-For "%{REMOTE_ADDR}s"
RequestHeader edit X-Forwarded-For ^([^,]*),?.*$ $1

# Logs management
LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" (%{ratio}n) \"%{User-Agent}i\" %D %{Correlation-ID}i %H" combined
ErrorLogFormat "[%t] [%m:%l] [pid %P:tid %T] [client\ %a] [referer\ %{Referer}i] [correlation_id\ %{Correlation-ID}i] %M -BT: %{error==}B - %{oserror==}B"

PidFile /usr/local/apache2/httpd.pid
Mutex file:/usr/local/apache2 default
DefaultRuntimeDir /usr/local/apache2

ScoreBoardFile /usr/local/apache2/logs/httpd.scoreboard
CustomLog "|| /usr/local/apache2/bin/rotatelogs -l /usr/local/apache2/logs/access_log.%Y-%m-%d-%H_%M_%S 3600" combined env=!dontlog
ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l /usr/local/apache2/logs/error_log.%Y-%m-%d-%H_%M_%S 60"

LogLevel debug

# Rewrite to redirect root URLs to the application URL
RewriteEngine on

RewriteRule "^/services/param/([0-9]+)/date/(.*)"    "/services/"

RewriteCond %{HTTPS} !=on
RewriteRule ^/(.*) [https://%{HTTP_HOST}/$1]https://%{HTTP_HOST}/$1 [R,L]

# Parametering for SSL
SSLRandomSeed startup builtin
AddType application/x-x509-ca-cert .crt
AddType application/x-pkcs7-crl    .crl
SSLPassPhraseDialog  builtin
SSLSessionCache "shmcb:/usr/local/apache2/logs/ssl_scache(512000)"
Mutex default
SSLEngine on

SSLCipherSuite ECDHE-#########-SHA256
# Active only TLS v1.1 & v1.2
SSLProtocol All -SSLv3
SSLCertificateFile /usr/local/apache2/secret/cert_pki_ent_HP.pem
SSLCertificateKeyFile /usr/local/apache2/secret/cert_pki_ent_HP.key
SSLCertificateChainFile /usr/local/apache2/secret/cert_pki_ent_HP.cer
SSLHonorCipherOrder     off

SSLProxyEngine on

SetEnvIf User-Agent ".*MSIE.*" \
    nokeepalive ssl-unclean-shutdown \
    downgrade-1.0 force-response-1.0

Customlog "||/usr/local/apache2/bin/rotatelogs -l //usr/local/apache2/logs/ssl_request_log.%Y-%m-%d-%H_%M_%S 3600" "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"

# Must be outside <location></location>
Options +FollowSymLinks

############################################
# Customer Cache Treatment
############################################
RequestHeader  edit "If-None-Match" "^\"(.*)-gzip\"$" "\"$1\""
Header  edit "ETag" "^\"(.*[^g][^z][^i][^p])\"$" "\"$1-gzip\""
ExpiresActive On
ExpiresDefault "access plus 0 minutes"
ExpiresByType text/html "access plus 0 minutes"
ExpiresByType image/gif "access plus 24 hours"
ExpiresByType image/jpeg "access plus 24 hours"
ExpiresByType image/png "access plus 24 hours"
ExpiresByType text/css "access plus 24 hours"
ExpiresByType text/javascript "access plus 24 hours"
ExpiresByType application/javascript "access plus 24 hours"
ExpiresByType image/svg+xml "access plus 24 hours"

<Location />
  SetOutputFilter DEFLATE
  BrowserMatch ^Mozilla/4 gzip-only-text/html
  BrowserMatch ^Mozilla/4\.0[678] no-gzip
  BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
  SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla)$ no-gzip dont-vary
  Header append Vary User-Agent env=!dont-vary
</Location>

# Rewrite to redirect root URLs to the application URL
Header edit Set-Cookie "^(API_JSESSIONID=.*); .*" "$1; domain=.example.com; HttpOnly; secure"
Header edit Set-Cookie (XSRF-TOKEN=.*);.* "$1; path=/; domain=.example.com; secure"

ExtendedStatus On

# The server returns Server: Apache
ServerTokens productOnly
# Returns an error code 405 (unauthorized method) to the client (mod_proxy)
TraceEnable off

ServerSignature off
HostnameLookups off

# Security Restriction: Cache Apache and prevents the use of trace mode
RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
RewriteRule .* - [F]

# Parameters for validation Geomobile Client Certificate
SSLVerifyDepth          2
SSLCACertificateFile    /usr/local/apache2/secret/chain_bundle.pem
SSLVerifyClient none

<VirtualHost *:443>

    SSLEngine on

    ServerName api.example.com
    ServerAlias api.example.com

    <IfModule proxy_hcheck_module>
        LogLevel proxy_hcheck:trace8 proxy:trace8
    </IfModule>

    # Server HTTPD
    DocumentRoot /usr/local/apache2/htdocs/test

    SetEnvIf Origin "http(s)?://(www\.)?(admin.example.com:443|home.example.com:443|warnings.example.com:443|files.example.com:443|hook.example.com:443|optional.example.com:443|review.example.com:443|api.example.com:443|cluster.example.com:54657|cluster.example.com:54656|cluster.example.com:54658|cluster.example.com:54655|cluster.example.com:54659|cluster.example.com:54660|cluster.example.com:54661)$" AccessControlAllowOrigin=$0
    Header set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin
    Header set Access-Control-Allow-Credentials "true"
    # CSRF
    RequestHeader unset Access-Control-Request-Headers
    Header set Access-Control-Allow-Headers "X-XSRF-TOKEN, content-type"
    Header set Access-Control-Expose-Headers Circuit-Breakers-On-Error

    CustomLog "|| /usr/local/apache2/bin/rotatelogs -p '/usr/local/apache2/outils/MANAGE_LOGS.sh' -l //usr/local/apache2/logs/access_test_log.%Y-%m-%d-%H_%M_%S 3600" combined env=!dontlog
    ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l //usr/local/apache2/logs/error_test_log.%Y-%m-%d-%H_%M_%S 60"

    ProxyHCExpr reqok {%{REQUEST_STATUS} =~ /^[2]/}

    <Proxy balancer://application-files/>
        BalancerMember http://server-one:9382 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
        BalancerMember http://server-two:9382 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
    </Proxy>

    <Proxy balancer://application-db/>
        BalancerMember http://server-one:9385 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
        BalancerMember http://server-two:9385 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
    </Proxy>

    <Proxy balancer://application-jobs/>
        BalancerMember http://server-one:9384 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
        BalancerMember http://server-two:9384 ttl=30 connectiontimeout=2 Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
    </Proxy>

    # Balancer manager
    ProxyPass /balancer-manager !
    <Location /balancer-manager>
        SetHandler balancer-manager
        Require host server-one.example.com server-two.example.com server-one server-two
    </Location>

    ProxyPass /config.api.json !
    ProxyPass /config-extra.api.json !
    ProxyPass /version.json !
    ProxyPassMatch /management.* !

    ProxyPass /services/ balancer://application-files/services/
    ProxyPassReverse /services/ balancer://application-files/services/

    ProxyPass /db_services/ balancer://application-db/db_services/
    ProxyPassReverse /db_services/ balancer://application-db/db_services/

    ProxyPass /jobs/ balancer://application-jobs/jobs/
    ProxyPassReverse /jobs/ balancer://application-jobs/jobs/

</VirtualHost>
```
In the logs I can't find the reason stopping the performance of the HealthCheck and the Balancer.

Could someone help me find the possible source of the error or how to extract more information to report it as a bug, plis.

Thank in advance.