On Mon, Oct 06, 2008 at 01:07:49PM -0700, Gordon Mohr wrote:
> I can't find mention of this '-I' option elsewhere. (It's not in my
> 2.6.STABLE14-based man page.)
>
> Is there a writeup on this option anywhere?
>
> Did it only appear in later versions?
Right, sorry, it appeared in 2.7:
http://www.squid-cache.org/cgi-bin/cvsweb.cgi/squid/doc/squid.8.in
> Is there a long-name for the option that would be easier to search for?
No.
> I would be interested in seeing your scripts if there are other wrinkles
> to using Squid in this manner. We're currently using squid for
> load-balancing on dedicated dual-core machines, so one core is staying
> completely idle...
I'm including a perl script called 'multisquid' below that uses -I and
assumes that there are '.squid-N.conf' configure scripts where "N" is a
number 0, 1, etc. I'm also including a bash script 'init-squid' that
generates those from a squid.conf based on the number of subdirectories
under the cache_dir of the form 0, 1, etc (up to 4) exist. It makes
squid 0 a cache_peer parent of the others so it's the only one that
makes upstream connections, but they all can serve clients.
- Dave
> Dave Dykstra wrote:
> >Meanwhile the '-I' option to squid makes it possible to run multiple
> >squids serving the same port on the same machine, so you can make use of
> >more CPUs. I've got scripts surrounding squid startups to take
> >advantage of that. Let me know if you're interested in having them.
> >Currently I run a couple machines using 2 squids each on 2 bonded
> >gigabit interfaces in order to get over 200 Mbytes/second throughput.
> >
> >- Dave
------------------------------ multisquid ---------------------------
#!/usr/bin/perl -w
#
# run multiple squids.
# If the command line options are for starting up and listening on a
# socket, first open a socket for them to share with squid -I.
# If either one results in an error exit code, return the first error code.
# Writtten by Dave Dykstra, July 2007
#
use strict;
use Socket;
use IO::Handle;
use Fcntl;
if ($#ARGV < 2) {
print STDERR "Usage: multisquid squidprefix numsquids http_port [squid_args ...]\n";
exit 2;
}
my $prefix = shift(@ARGV);
my $numsquids = shift(@ARGV);
my $port = shift(@ARGV);
my $proto = getprotobyname('tcp');
if (!(($#ARGV >= 0) && (($ARGV[0] eq "-k") || ($ARGV[0] eq "-z")))) {
#open the socket for both squids to listen on if not doing an
# operation that doesn't use the socket (that is, -k or -z)
close STDIN;
my $fd;
socket($fd, PF_INET, SOCK_STREAM, $proto) || die "socket: $!";
setsockopt($fd, SOL_SOCKET, SO_REUSEADDR, 1) || die "setsockopt: $!";
bind($fd, sockaddr_in($port, INADDR_ANY)) || die "bind of port $port: $!";
}
my $childn;
for ($childn = 0; $childn < $numsquids; $childn++) {
if (fork() == 0) {
exec "$prefix/sbin/squid -f $prefix/etc/.squid-$childn.conf -I @ARGV" || die "exec: $!";
}
# get them to start at different times so they're identifiable by squidclient
sleep 2;
}
my $exitcode = 0;
while(wait() > 0) {
if (($? != 0) && ($exitcode == 0)) {
# Take the first non-zero exit code and ignore the other one.
# exit expects a byte, but the exit code from wait() has signal
# numbers in low byte and exit code in high byte. Combine them.
$exitcode = ($? >> 8) | ($? & 255);
}
}
exit $exitcode;
------------------------------ init-squid ---------------------------
#!/bin/bash
# This script will work with one squid or up to 4 squids on the same http port.
# The number of squids is determined by the existence of cache directories
# as follows. The main path to the cache directories is determined by the
# cache_dir option in squid.conf. To run multiple squids, create directories
# of the form
# `dirname $cache_dir`/$N/`basename $cache_dir`
# where N goes from 0 to the number of squids minus 1. Also create
# cache_log directories of the same form. Note that the cache_log option
# in squid.conf is a file, not a directory, so the $N is up one level:
# cache_log_file=`basename $cache_log`
# cache_log_dir=`dirname $cache_log`
# cache_log_dir=`dirname $cache_log_dir`/$N/`basename $cache_log_dir`
# The access_log should be in the same directory as the cache_log, and
# the pid_filename also needs to be in similarly named directories (the
# same directories as the cache_log is a good choice).
. /etc/init.d/functions
RETVAL=0
INSTALL_DIR=_your_base_install_dir_with_squid_and_utils_subdirectories_
#size at which rotateiflarge will rotate access.log
LARGE_ACCESS_LOG=1000000000
CONF_FILE=$INSTALL_DIR/squid/etc/squid.conf
CACHE_DIR=`awk '$1 == "cache_dir" {x=$3} END{print x}' $CONF_FILE`
CACHE_LOG=`awk '$1 == "cache_log" {x=$2} END{print x}' $CONF_FILE`
ACCESS_LOG=`awk '$1 == "access_log" {x=$2} END{print x}' $CONF_FILE`
squid_dirs()
{
# if $NUMSQUIDS is 1, echo the parameter, otherwise echo the parameter
# N times with $N before the basename of the parameter, where N is
# from 0 to $NUMSQUIDS-1
if [ $NUMSQUIDS = 1 ]; then
echo $1
return
fi
typeset N
N=0
while [ $N -lt $NUMSQUIDS ]; do
echo `dirname $1`/$N/`basename $1`
let N=$N+1
done
}
#see how many squid cache directories exist (up to 4)
NUMSQUIDS=0
for D in `NUMSQUIDS=4 squid_dirs $CACHE_DIR`; do
if [ ! -d $D ]; then
break
fi
let NUMSQUIDS=$NUMSQUIDS+1
done
if [ $NUMSQUIDS = 0 ]; then
NUMSQUIDS=1
SQUID=$INSTALL_DIR/squid/sbin/squid
PLURAL=""
else
PLURAL="s"
# create the squid.conf files for each squid
PID_FILENAME=`awk '$1 == "pid_filename" {x=$2} END{print x}' $CONF_FILE`
HTTP_PORT=`awk '$1 == "http_port" {print $2;exit}' $CONF_FILE`
SNMP_PORT=`awk '$1 == "snmp_port" {x=$2} END{print x}' $CONF_FILE`
ICP_PORT=`awk '$1 == "icp_port" {x=$2} END{print x}' $CONF_FILE`
ICP_PORT=`awk '$1 == "icp_port" {x=$2} END{print x}' $CONF_FILE`
#visible_hostname is the name that shows up in X-Cache and Via headers
VISIBLE_HOSTNAME=`awk '$1 == "visible_hostname" {x=$2} END{print x}' $CONF_FILE`
HOSTNAME=`hostname`
SQUID="$INSTALL_DIR/utils/sbin/multisquid $INSTALL_DIR/squid $NUMSQUIDS ${HTTP_PORT:-3128}"
CACHEBASE="$(dirname $CACHE_DIR)"
LOGBASE="$(dirname $(dirname $CACHE_LOG))"
PIDBASE="$(dirname $(dirname $PID_FILENAME))"
N=0
while [ $N -lt $NUMSQUIDS ]; do
SEDCMDS="-e \"s,$CACHEBASE/,$CACHEBASE/$N/,\""
if [ "$LOGBASE" != "$CACHEBASE" ]; then
SEDCMDS="$SEDCMDS -e \"s,$LOGBASE/,$LOGBASE/$N/,\""
fi
if [ "$PIDBASE" != "$CACHEBASE" ] && [ "$PIDBASE" != "$LOGBASE" ]; then
SEDCMDS="$SEDCMDS -e \"s,$PIDBASE/,$PIDBASE/$N/,\""
fi
NEWCONF=$INSTALL_DIR/squid/etc/.squid-$N.conf
rm -f $NEWCONF
eval sed $SEDCMDS $INSTALL_DIR/squid/etc/squid.conf | awk '
BEGIN{print "# DO NOT EDIT -- Automatically generated by '$0'";print}
/^[ \t#]*http_port[ \t]/ {
if (!got_http_port) {
#on first squid, add additional http_port after first one specified,
# or after first commented http_port if none specified
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$HTTP_PORT'" == "") {portnum=3128; print}
if (portnum != 0) {
print "#first port overridden by command line -I"
print "http_port", portnum
# have each squid also listen on their own port, the first one to
# be parent of others and the others for forced cache reloads
print "http_port", portnum - ('$N' + 1)
got_http_port=1
next
}
}
else if ((substr($0,1,1) != "#") && ("'$N'" != 0)) {
# only first squid listens on other http ports, skip this one
next
}
}
/^[ \t#]*snmp_port[ \t]/ {
#replace snmp_port with separate number for each squid
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$SNMP_PORT'" == "") {portnum=3401; print}
if (portnum != 0) {
portnum=portnum + '$N'
print "snmp_port", portnum
next
}
}
/^[ \t#]*icp_port[ \t]/ {
#only first squid listens on icp port
portnum=0
if (substr($0,1,1) != "#") portnum=$2
else if ("'$SNMP_PORT'" == "") {portnum=3130; print}
if (portnum != 0) {
if ("'$N'" != 0) portnum=0 # disables the icp port
print "icp_port", portnum
next
}
}
/^[ \t#]*cache_peer[ \t]/ {
#any squid after first one go only to first one as a cache_peer parent
if ("'$N'" != 0) {
if (substr($0,1,1) == "#") {
if (!got_cache_peer) {
#insert the line after the first commented-out one
print
portnum='${HTTP_PORT:-3128}' - 1
print "cache_peer localhost parent", portnum, "0 no-query"
got_cache_peer=1
next
}
}
else {
#skip any other uncommented cache_peer options on non-first squids
next
}
}
}
$1 == "visible_hostname" || /TAG: visible_hostname/ {
hostname=""
if (substr($0,1,1) != "#") hostname=$2
else if ("'$VISIBLE_HOSTNAME'" == "") {hostname="'$HOSTNAME'"; print}
if (hostname != "") {
print "visible_hostname",hostname "/'$N'"
}
}
{print}
' >$NEWCONF
chmod a-w $NEWCONF
let N=$N+1
done
fi
last_logline_has()
{
# if first parameter is 1, see if the last line of any of the cache logs
# of squids contains the other parameters; if the first parameter is 0,
# see if true for all of the cache logs
typeset ANY LDIR LFILE
ANY=$1
shift
for LDIR in $(squid_dirs $(dirname $CACHE_LOG)); do
LFILE=$LDIR/`basename $CACHE_LOG`
if tail -1 $LFILE | egrep -q "$*"; then
if [ "$ANY" = 1 ]; then
# found one match, return true
return 0
fi
else
if [ "$ANY" = 0 ]; then
# found one non-match, return false
return 1
fi
fi
done
# return true for ANY=0 (all matched), false for ANY=1 (none matched)
return $ANY
}
start_squid()
{
echo "Starting $NUMSQUIDS Squid${PLURAL}... "
$SQUID -DFS
# it can take a while to check out a large cache
LIMIT=60 # Number of Tries
a=1
while [ $a -le "$LIMIT" ]
do
a=$(($a+1))
sleep 10
$SQUID -k check
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid start failed!!!"
start_squid_fail
RETVAL=$?
break
else
if last_logline_has 0 "storeLateRelease: released" ;then
break
RETVAL=0
fi
fi
done
sleep 2
echo "done."
[ $RETVAL == 0 ]
}
start_squid_fail()
{
RETVAL=0
$SQUID -k shutdown 2>/dev/null|| true # in case all are not shut down
if last_logline_has 1 "(store_errors|faults)" ;then
echo "Clearing Caches and Restarting $NUMSQUIDS Squid${PLURAL}... "
for D in `squid_dirs $CACHE_DIR`; do
rm -rf $D/*
done
$SQUID -z
sleep 3
$SQUID -DFS
sleep 3
$SQUID -k check
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid start failed!!!"
$SQUID -k shutdown 2>/dev/null || true # in case all are not shut down
fi
else
echo "Script Can't Fix Squid Problem"
RETVAL=1
fi
[ $RETVAL == 0 ]
}
stop_squid()
{
echo -n "Stopping $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k shutdown
RETVAL=$?
if [ $RETVAL != 0 ] ; then
echo "Squid stop failed!!!"
else
sleep 30
echo "done."
fi
[ $RETVAL == 0 ]
}
cleancache()
{
echo "Clearing Caches of $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k shutdown 2>/dev/null||true # in case all are not shut down
for D in `squid_dirs $CACHE_DIR`; do
rm -rf $D/*
done
$SQUID -z
}
rotate()
{
echo "Rotating Caches of $NUMSQUIDS Squid${PLURAL}... "
$SQUID -k rotate
}
rotateiflarge()
{
for LDIR in $(squid_dirs $(dirname $ACCESS_LOG)); do
LFILE=$LDIR/`basename $ACCESS_LOG`
if [ "$(stat -c %s $LFILE)" -gt $LARGE_ACCESS_LOG ]; then
rotate
break
fi
done
}
start()
{
start_squid
[ $RETVAL == 0 ]
}
stop()
{
stop_squid
[ $RETVAL == 0 ]
}
case "$1" in
start)
cleancache
start
;;
stop)
stop
;;
status)
$SQUID -k check 2>/dev/null
exit $?
;;
reload)
$SQUID -k reconfigure
;;
restart)
stop
start
;;
cleancache)
cleancache
;;
rotate)
rotate
;;
rotateiflarge)
rotateiflarge
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|cleancache|rotate|rotateiflarge}"
exit 1
esac
