High Disk I/O at 1.1Gbps

Hi All,

This is my first post. So please forgive me if I make any mistake. I have 3 server all of them Dell R730 servers all of them running:
CentOS 7
nginx version: nginx/1.12.2
PHP 7.1.27 (cli) (built: Mar 10 2019 16:23:55) ( NTS )
The website shows basic directory indexing using _h5ai
Clients basically browse through the index and download movie.
I have 10G Network Card
RAID 6 configured with 4TB x 12pc SAS HD = 37 TB Usable (XFS Format)
My OS is in separate 256 GB SSD disk.

My problem is when ever the bandwidth utilization gets near 1.1 Gbps or above the RAID 6 Drive shows 100% or 101% busy. I have used atop command to check it. Screen shot also given below:

atop command output:
DSK | sda | busy 101% | read 9686 | write 5 | KiB/r 128 | KiB/w 13 | MBr/s 121.6 | MBw/s 0.0 | avq 71.04 | avio 1.03 ms |



[root@node1 ~]# ethtool p5p2
Settings for p5p2:
Supported ports: [ FIBRE ]
Supported link modes: 10000baseT/Full
Supported pause frame use: Symmetric
Supports auto-negotiation: No
Supported FEC modes: Not reported
Advertised link modes: 10000baseT/Full
Advertised pause frame use: Symmetric
Advertised auto-negotiation: No
Advertised FEC modes: Not reported
Speed: 10000Mb/s
Duplex: Full
Port: Direct Attach Copper
PHYAD: 0
Transceiver: internal
Auto-negotiation: off
Supports Wake-on: d
Wake-on: d
Current message level: 0x00000007 (7)
drv probe link
Link detected: yes

[root@node1 ~]# cat /etc/nginx/nginx.conf
# ---------------------------------------------------------------------
# NGINX - FastCGI CACHE configuration
# ---------------------------------------------------------------------
# Created by Ryadel on 2017.12.09
# www.ryadel.com
# ---------------------------------------------------------------------

user nginx;

worker_processes 38;
working_directory /var/www/html;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
worker_connections 1024;
accept_mutex off;
}

http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

access_log /var/log/nginx/access.log main;
#access_log off;

sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;

include /etc/nginx/mime.types;
default_type application/octet-stream;

# ---------------------------------------------------------------------
# FASTCGI GLOBAL CONFIGURATION - START
# ---------------------------------------------------------------------
# below lines must be outside server blocks to enable FastCGI cache for Nginx
# path can be anywhere, app name must be consistent, total size small enough to avoid RAM depletion

fastcgi_cache_path /var/cache/nginx/fastcgi levels=1:2 keys_zone=WORDPRESS:256m inactive=30m;
fastcgi_cache_key "$scheme$request_method$host$request_uri";
fastcgi_cache_use_stale error timeout invalid_header http_500;
fastcgi_ignore_headers Cache-Control Expires Set-Cookie;

# ---------------------------------------------------------------------
# FASTCGI GLOBAL CONFIGURATION - END
# ---------------------------------------------------------------------

# Custom header that will tell us if the response page has been served from cache (HIT) or from the upstream host (BYPASS or MISS).
add_header X-FastCGI-Cache $upstream_cache_status;

server {
listen 80;
server_name default_server;

root /var/www/html;
index index.html index.php /_h5ai/public/index.php;
autoindex off;
gzip on;
gzip_types application/javascript image/* text/css;
gunzip on;
location ~* \.(jpg|jpeg|png|gif|ico)$ {
expires 30d;
}
location ~* \.(css|js)$ {
expires 7d;
}
set $skip_cache 0;

# ---------------------------------------------------------------------
# CACHE SKIP RULES - START
# ---------------------------------------------------------------------

# Do not cache POST requests - they should always go to PHP
if ($request_method = POST) {
set $skip_cache 1;
}

# Do not cache URLs with a query string - they should always go to PHP
if ($query_string != "") {
set $skip_cache 1;
}

# WooCommerce-specific cache skip rules
if ($request_uri ~* "/store.*|/cart.*|/my-account.*|/checkout.*|/addons.*") {
set $skip_cache 1;
set $skip_cache_reason WP_WooCommerce;
}

if ($cookie_woocommerce_items_in_cart) {
set $skip_cache 1;
set $skip_cache_reason WP_WooCommerce;
}

if ($request_uri ~* ("/cart.*")) {
set $skip_cache 1;
}

# Don't cache URIs containing the following segments (admin panel, sitemaps, feeds, etc.)
if ($request_uri ~* "(/wp-admin/|/xmlrpc.php|/wp-(app|cron|login|register|mail).php|wp-.*.php|/feed/|index.php|wp-comments-popup.php|wp-links-opml.php|wp-locations.php|sitemap(_index)?.xml|[a-z0-9_-]+-sitemap([0-9]+)?.xml)") {
set $skip_cache 1;
}

# Don't use the cache for logged-in users or recent commenters
if ($http_cookie ~* "comment_author|wordpress_[a-f0-9]+|wp-postpass|wordpress_no_cache|wordpress_logged_in") {
set $skip_cache 1;
}

# ---------------------------------------------------------------------
# CACHE SKIP RULES - END
# ---------------------------------------------------------------------

location / {
try_files $uri $uri/ /index.php?$args;
}

location ~ \.php$ {

try_files $uri =404; # comment out this line if php-fpm is hosted on a remote machine
include /etc/nginx/fastcgi.conf;

#limit_req zone=limit_req_php burst=20 nodelay;

fastcgi_pass 127.0.0.1:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root/$fastcgi_script_name;
fastcgi_cache WORDPRESS;
fastcgi_cache_valid 200 60m;
fastcgi_cache_bypass $skip_cache;
fastcgi_no_cache $skip_cache;
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f $document_root$fastcgi_script_name) {
return 404;
}
}
}
}

[root@node1 ~]# cat /etc/nginx/fastcgi.conf

fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;

fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param REQUEST_SCHEME $scheme;
fastcgi_param HTTPS $https if_not_empty;

fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;

fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;

# PHP only, required if PHP was built with --enable-force-cgi-redirect
fastcgi_param REDIRECT_STATUS 200;

[root@node1 ~]# free -g
total used free shared buff/cache available
Mem: 125 1 0 2 123 120
Swap: 3 0 3