#!/usr/bin/perl
# store rewrite originally writen by chudy_fernandez@yahoo.com
# modified by member of comstuff.net to satisfy common and dynamic content.
#
$|=1;
while (<>) {
@X = split;
# $X[1] =~ s/&sig=.*//;
$x = $X[0] . " ";
$_ = $X[1];
$u = $X[1];
#speedtest
if (m/^http:\/\/(.*)\/speedtest\/(.*\.(jpg|txt))\?(.*)/) {
print $x . "http://www.speedtest.net.SQUIDINTERNAL/speedtest/" . $2 . "\n";
#mediafire
}elsif (m/^http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)/) {
print $x . "http://www.mediafire.com.SQUIDINTERNAL/" . $1 ."/" . $2 . "\n";
#fileserve
}elsif (m/^http:\/\/fs\w*\.fileserve\.com\/file\/(\w*)\/[\w-]*\.\/(.*)/) {
print $x . "http://www.fileserve.com.SQUIDINTERNAL/" . $1 . "./" . $2 . "\n";
#filesonic
}elsif (m/^http:\/\/s[0-9]*\.filesonic\.com\/download\/([0-9]*)\/(.*)/) {
print $x . "http://www.filesonic.com.SQUIDINTERNAL/" . $1 . "\n";
#4shared
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/download\/(.*)\/(.*\..*)\?.*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/download/$2\/$3\n";
#4shared preview
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/img\/(\d*)\/\w*\/dlink__2Fdownload_2F(\w*)_3Ftsid_3D[\w-]*\/preview\.mp3\?sId=\w*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/$2\n";
#photos-X.ak.fbcdn.net where X a-z
}elsif (m/^http:\/\/photos-[a-z](\.ak\.fbcdn\.net)(\/.*\/)(.*\.jpg)/) {
print $x . "http://photos" . $1 . "/" . $2 . $3 . "\n";
#YX.sphotos.ak.fbcdn.net where X 1-9, Y a-z
} elsif (m/^http:\/\/[a-z][0-9]\.sphotos\.ak\.fbcdn\.net\/(.*)\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 ."/". $2 . "\n";
#maps.google.com
} elsif (m/^http:\/\/(cbk|mt|khm|mlt|tbn)[0-9]?(.google\.co(m|\.uk|\.id).*)/) {
print $x . "http://" . $1 . $2 . "\n";
# compatibility for old cached get_video?video_id
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com).*?(videoplayback\?id=.*?|video_id=.*?)\&(.*?)/) {
$z = $2; $z =~ s/video_id=/get_video?video_id=/;
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $z . "\n";
#====
# Video Youtube
# acl dontrewrite url_regex redbot\.org (get_video|videoplayback\?id|videoplayback.*id).*begin\=[1-9][0-9]*
# acl store_rewrite_list url_regex -i (youtube|google).*(videoplayback|liveplay)
# refresh_pattern -i (youtube|google).*(videoplayback|liveplay) 1440 99% 14400 override-expire override-lastmod ignore-no-cache ignore-private reload-into-ims ignore-must-revalidate ignore-reload store-stale
} elsif ($X[1] =~ m/^http(|s)\:\/\/.*youtube.*(ptracking|stream_204|player_204|gen_204).*(video_id|docid|v)\=([^\&\s]*).*/){
$vid = $4 ;
@cpn = m/[&?]cpn\=([^\&\s]*)/;
if (defined($vid )) {
$fn = "/var/squid/log/@cpn";
unless (-e $fn) {
open FH,">".$fn ;
print FH "$vid\n";
close FH;
}
}
print $x . $X[1] . "\n";
} elsif ($X[1] =~ m/^http:\/\/.*(youtube|google).*videoplayback.*/){
@itag = m/[&?](itag=[\d]*)/;
@ids = m/[&?]id\=([^\&\s]*)/;
@mime = m/[&?](mime\=[^\&\s]*)/;
@cpn = m/[&?]cpn\=([^\&\s]*)/;
if (defined($cpn[0])) {
$fn = "/var/squid/log/@cpn";
if (-e $fn) {
open FH,"<".$fn ;
$id = <FH>;
chomp $id ;
close FH ;
} else {
$id = $ids[0] ;
}
} else {
$id = $ids[0] ;
}
@range = m/[&?](range=[^\&\s]*)/;
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/id=" . $id . "&@itag@range@mime\n";
#====
} elsif (m/^http:\/\/www\.google-analytics\.com\/__utm\.gif\?.*/) {
print $x . "http://www.google-analytics.com/__utm.gif\n";
#Cache High Latency Ads
} elsif (m/^http:\/\/([a-z0-9.]*)(\.doubleclick\.net|\.quantserve\.com|\.googlesyndication\.com|yieldmanager|cpxinteractive)(.*)/) {
$y = $3;$z = $2;
for ($y) {
s/pixel;.*/pixel/;
s/activity;.*/activity/;
s/(imgad[^&]*).*/\1/;
s/;ord=[?0-9]*//;
s/;×tamp=[0-9]*//;
s/[&?]correlator=[0-9]*//;
s/&cookie=[^&]*//;
s/&ga_hid=[^&]*//;
s/&ga_vid=[^&]*//;
s/&ga_sid=[^&]*//;
# s/&prev_slotnames=[^&]*//
# s/&u_his=[^&]*//;
s/&dt=[^&]*//;
s/&dtd=[^&]*//;
s/&lmt=[^&]*//;
s/(&alternate_ad_url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&ref=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&cookie=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/[;&?]ord=[?0-9]*//;
s/[;&]mpvid=[^&;]*//;
s/&xpc=[^&]*//;
# yieldmanager
s/\?clickTag=[^&]*//;
s/&u=[^&]*//;
s/&slotname=[^&]*//;
s/&page_slots=[^&]*//;
}
print $x . "http://" . $1 . $2 . $y . "\n";
#cache high latency ads
} elsif (m/^http:\/\/(.*?)\/(ads)\?(.*?)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
# spicific servers starts here....
} elsif (m/^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*?)/) {
print $x . "http://" . $1 . "\n";
#cdn, varialble 1st path
} elsif (($u =~ /filehippo/) && (m/^http:\/\/(.*?)\.(.*?)\/(.*?)\/(.*)\.([a-z0-9]{3,4})(\?.*)?/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/[a-z0-9]{2,5}/cdn./;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#rapidshare
} elsif (($u =~ /rapidshare/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?)([a-z]*\.[^\/]{3}\/[a-z]*\/[0-9]*)\/(.*?)\/([^\/\?\&]{4,})$/)) {
print $x . "http://cdn." . $3 . "/SQUIDINTERNAL/" . $5 . "\n";
} elsif (($u =~ /maxporn/) && (m/^http:\/\/([^\/]*?)\/(.*?)\/([^\/]*?)(\?.*)?$/)) {
print $x . "http://" . $1 . "/SQUIDINTERNAL/" . $3 . "\n";
#like porn hub variables url and center part of the path, filename etention 3 or 4 with or without ? at the end
} elsif (($u =~ /tube8|pornhub|xvideos/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?(\.[a-z]*)?)\.([a-z]*[0-9]?\.[^\/]{3}\/[a-z]*)(.*?)((\/[a-z]*)?(\/[^\/]*){4}\.[^\/\?]{3,4})(\?.*)?$/)) {
print $x . "http://cdn." . $4 . $6 . "\n";
#...spicific servers end here.
#photos-X.ak.fbcdn.net where X a-z
} elsif (m/^http:\/\/photos-[a-z].ak.fbcdn.net\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 . "\n";
#for yimg.com video
} elsif (m/^http:\/\/(.*yimg.com)\/\/(.*)\/([^\/\?\&]*\/[^\/\?\&]*\.[^\/\?\&]{3,4})(\?.*)?$/) {
print $x . "http://cdn.yimg.com//" . $3 . "\n";
#for yimg.com doubled
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*?)\.yimg\.com\/(.*?)\?(.*)/) {
print $x . "http://cdn.yimg.com/" . $3 . "\n";
#for yimg.com with &sig=
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*)/) {
@y = ($1,$2);
$y[0] =~ s/[a-z]+[0-9]+/cdn/;
$y[1] =~ s/&sig=.*//;
print $x . "http://" . $y[0] . ".yimg.com/" . $y[1] . "\n";
#youjizz. We use only domain and filename
} elsif (($u =~ /media[0-9]{2,5}\.youjizz/) && (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\/([^\/\?\&]*)\.([^\/\?\&]{3,4})((\?|\%).*)?$/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#general purpose for cdn servers. add above your specific servers.
} elsif (m/^http:\/\/([0-9.]*?)\/\/(.*?)\.(.*)\?(.*?)/) {
print $x . "http://squid-cdn-url//" . $2 . "." . $3 . "\n";
#generic http://variable.domain.com/path/filename."ex" "ext" or "exte" with or withour "? or %"
} elsif (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\.([^\/\?\&]{2,4})((\?|\%).*)?$/) {
@y = ($1,$2,$3,$4);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
# generic http://variable.domain.com/...
} elsif (m/^http:\/\/(([A-Za-z]+[0-9-]+)*?|.*cdn.*|.*cache.*)\.(.*?)\.(.*?)\/(.*)$/) {
print $x . "http://cdn." . $3 . "." . $4 . "/" . $5 . "\n";
# spicific extention that ends with ?
} elsif (m/^http:\/\/(.*?)\/(.*?)\.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|on2)(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "." . $3 . "\n";
# all that ends with ;
} elsif (m/^http:\/\/(.*?)\/(.*?)\;(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
} else {
print $x . $_ . "sucks\n";
}
}
Saturday, 29 March 2014
FSENSE 2.1 logs
PFSENSE 2.1 logs berwarna dengan CCZE
ON 07.190 KOMENTAR
Cara Instal dan Console Putty
Untuk i386/x86
pkg_add -r http://ftp.freebsd.org/pub/FreeBSD/ports/i386/packages-8-stable/All/ccze-0.2.1_4.tbz
Untuk amd64
pkg_add -r http://ftp.freebsd.org/pub/FreeBSD/ports/amd64/packages-8-stable/All/ccze-0.2.1_4.tbz
untuk menggunakannnya
#tail -f /var/squid/logs/access.log |ccze
Hasilnya :
Friday, 28 March 2014
Lusca Patch 2014 ( Youtube Cache)
Lusca Patch 2014 ( Youtube Cache)
POSTED BY ARDIANVIRANTO ARDIAN ON 03.010 KOMENTAR
1. Install pfsense2.0.3
http://files.nyi.pfsense.org/mirror/downloads/old/pfSense-LiveCD-2.0.3-RELEASE- i386-20130412-1022.iso.gz
2. Install lusca pada pfsense2.0.3
3. Upgrade pfsense secara online
4. Reboot.
5. Re-install lusca di Pfsense 2.1
serta cek file-file di dalam squid (include.conf, storeurl.conf, dll)
6. Konfigurasi ulang melalui webconfig-->service-->proxyserver
7. Restart lusca melalui --->status-->services-->pilih lusca
8. Harusnya Jalan bila tidak jalan ulang ke step 5.
Bila Youtube tidak jalan bisa ikuti langkah sebagai berikut :
dari konsole/putty
#pkg_add -r -v -f perl
#pkg_add -r -v -f p5-File-ReadBackwardsSelanjutnya ikuti langkah dibawah ini :
#cd /root
/ buat folder text documen copy file dibawah taruh di root dengan jdl sprti dbwh
buat file cobaReadBack.pl
isinya :
#-------------------------------
#!/usr/bin/perl
use File::ReadBackwards;
$fh = File::ReadBackwards->new('/var/squid/logs/access.log') or
die "can't read file: $!\n";
while ( defined($line = $fh->readline) )
{
print $line ;
}
#------------------------------- Attribut file
#chmod 0755 cobaReadBack.pl
cara tes atau menjalankannya :
# perl cobaReadBack.pl
-----------------------------
Perhatian.. folder log adalah : /var/squid/logs
----------------------------
Instal ccze supaya report nya berwarna :
#pkg_add -r -v -f ccze
2.Backup file /user/local/etc/squid/include.conf dan hapus semua isinya dan
paste kan teks dibawah ini :
#-----------------------
#thanks to ucok_karnadi@yahoo.com
#debug_options ALL,3
strip_query_terms off
acl yutub url_regex -i .*youtube\.com\/.*$
acl yutub url_regex -i .*youtu\.be\/.*$
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%h %ru
logformat squid1 %{Referer}>h %ru
#logformat squid1 %>h %ru
access_log /var/squid/logs/yt.log squid1 yutub
acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
acl redirec urlpath_regex -i .*&ir=1&rr=12
cache deny redirec
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
#acl range url_regex -i .*youtube\.com\/videoplayback.*range\=.*$
#http_access deny range
#storeurl_access deny reddeny
#
# $Rev$
## LUSCA
acl speedtest url_regex ^http:\/\/.*\/speedtest\/.*
acl store_rewrite_list urlpath_regex .*\.ak\.fbcdn\.net\/
acl store_rewrite_list urlpath_regex http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)
acl store_rewrite_list urlpath_regex s[0-9]*\.filesonic\.com\/download\/.*
acl store_rewrite_list urlpath_regex [a-zA-Z]{2}[0-9]*\.4shared\.com\/download\/
acl store_rewrite_list urlpath_regex \/(watch\?|get_video\?|videodownload\?|videoplayback.*id)
#acl store_rewrite_list urlpath_regex \.youtube\.com\/videoplayback \.youtube\.com\/videoplay \.youtube\.com\/get_video\?
#acl store_rewrite_list urlpath_regex \.youtube\.[a-z][a-z]\/videoplayback \.youtube\.[a-z][a-z]\/videoplay \.youtube\.[a-z][a-z]\/get_video\?
acl store_rewrite_list urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|rar|cab)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|exe|cab)$
acl store_rewrite_list_domain_CDN url_regex \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.*
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[a-z]+[0-9]\.google\.co(m|\.id)
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/\.www[0-9][0-9]\.indowebster\.com\/(.*)(rar|zip|flv|wm(a|v)|3gp|mp(4|3)|exe|msi|avi|(mp(e?g|a|e|1|2|3|4))|cab|exe)
acl store_rewrite_list urlpath_regex \/(get_video|videoplayback\?id|videoplayback.*id) \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|swf)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|krf|exe)$
acl store_rewrite_list_domain_CDN url_regex (khm|mt)[0-9]?.google.co(m|\.id) streamate.doublepimp.com.*\.js\? photos-[a-z].ak.fbcdn.net \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.* yieldmanager cpxinteractive ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl dontrewrite url_regex redbot\.org (get_video|videoplayback\?id|videoplayback.*id).*begin\=[1-9][0-9]* \.php\?
acl getmethod method GET
storeurl_access allow speedtest
storeurl_access deny dontrewrite
storeurl_access deny !getmethod
storeurl_access allow store_rewrite_list_domain_CDN
storeurl_access allow store_rewrite_list
storeurl_access allow store_rewrite_list_domain
storeurl_access allow store_rewrite_list_path
storeurl_access deny reddeny
storeurl_access deny all
storeurl_rewrite_program /usr/local/etc/squid/storeurl.pl
storeurl_rewrite_children 16
storeurl_rewrite_concurrency 99
acl snmppublic snmp_community public
cachemgr_passwd none config reconfigure
#work around for fragment videos of msn
acl msnvideo url_regex QualityLevel.*Fragment
http_access deny msnvideo
#always_direct allow html
#cache_peer localhost parent 4001 0 carp login=PASS name=backend-1
max_stale 10 years
include /usr/local/etc/squid/refresh.conf
#acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
#upgrade_http0.9 deny shoutcast
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
#read_ahead_gap 0 KB
#ie_refresh on
reload_into_ims on
strip_query_terms off
deny_info TCP_RESET localnet
negative_dns_ttl 1 second
negative_ttl 1 second
snmp_port 3401
snmp_access allow snmppublic all
maximum_single_addr_tries 2
retry_on_error on
#n_aiops_threads 64
# server_http11 on
#request_header_max_size 128 KB
#reply_header_max_size 128 KB
#range_offset_limit 10 MB
vary_ignore_expire on
#client_db off # this needs to be on for acl maxconn to work
ipcache_size 4096
fqdncache_size 20
#tcp_recv_bufsize 64 KB
pipeline_prefetch on
#half_closed_clients off
# 0x10 no delay, 0x08 throughput, 0x04 reliability
# 0x10 10000 (minimize delay) Use delay metric
# 0x08 01000 (maximize throughput) Use default metric
# 0x04 00100 (maximize reliability) Use reliability metric
# 0x02 00010 (minimize monetary cost) Use cost metric
# dscp squidtos+ECN
# 56 0xE0 11100000
# 48 0xc0 11000000
# 08 0x20 00100000
# 32 0x80 10000000
# 16 0x40 01000000
#tcp_outgoing_tos 0x03 video
#tcp_outgoing_tos 0xb8 html
#tcp_outgoing_tos 0x20 images
#tcp_outgoing_tos 0x02 all
#zph_mode tos
#zph_local 0xb8
#zph_parent 0x08
#acl monitor url_regex avira
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%
#---------------------------------
3. Backup file /user/local/etc/squid/storeurl.pl dan hapus semua isinya dan paste kan teks dibawah ini :
#---------------------------------
#!/usr/bin/perl
# store rewrite originally writen by chudy_fernandez@yahoo.com
# modified by member of comstuff.net to satisfy common and dynamic content.
#
$|=1;
while (<>) {
@X = split;
# $X[1] =~ s/&sig=.*//;
$x = $X[0] . " ";
$_ = $X[1];
$u = $X[1];
#speedtest
if (m/^http:\/\/(.*)\/speedtest\/(.*\.(jpg|txt))\?(.*)/) {
print $x . "http://www.speedtest.net.SQUIDINTERNAL/speedtest/" . $2 . "\n";
#mediafire
}elsif (m/^http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)/) {
print $x . "http://www.mediafire.com.SQUIDINTERNAL/" . $1 ."/" . $2 . "\n";
#fileserve
}elsif (m/^http:\/\/fs\w*\.fileserve\.com\/file\/(\w*)\/[\w-]*\.\/(.*)/) {
print $x . "http://www.fileserve.com.SQUIDINTERNAL/" . $1 . "./" . $2 . "\n";
#filesonic
}elsif (m/^http:\/\/s[0-9]*\.filesonic\.com\/download\/([0-9]*)\/(.*)/) {
print $x . "http://www.filesonic.com.SQUIDINTERNAL/" . $1 . "\n";
#4shared
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/download\/(.*)\/(.*\..*)\?.*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/download/$2\/$3\n";
#4shared preview
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/img\/(\d*)\/\w*\/dlink__2Fdownload_2F(\w*)_3Ftsid_3D[\w-]*\/preview\.mp3\?sId=\w*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/$2\n";
#photos-X.ak.fbcdn.net where X a-z
}elsif (m/^http:\/\/photos-[a-z](\.ak\.fbcdn\.net)(\/.*\/)(.*\.jpg)/) {
print $x . "http://photos" . $1 . "/" . $2 . $3 . "\n";
#YX.sphotos.ak.fbcdn.net where X 1-9, Y a-z
} elsif (m/^http:\/\/[a-z][0-9]\.sphotos\.ak\.fbcdn\.net\/(.*)\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 ."/". $2 . "\n";
#maps.google.com
} elsif (m/^http:\/\/(cbk|mt|khm|mlt|tbn)[0-9]?(.google\.co(m|\.uk|\.id).*)/) {
print $x . "http://" . $1 . $2 . "\n";
# compatibility for old cached get_video?video_id
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com).*?(videoplayback\?id=.*?|video_id=.*?)\&(.*?)/) {
$z = $2; $z =~ s/video_id=/get_video?video_id=/;
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $z . "\n";
# youtube fix by th30nly @comstuff.net, ucok_karnadi@yahoo.com, extreemblank@yahoo.com
# for ALL Youtube ( range & non range )
# first you need do this
# install package dependencies "apt-get install libfile-readbackwards-perl"
# add line below to your squid config and remove "#"
# strip_query_terms off
# acl yutub url_regex -i .*youtube\.com\/.*$
# acl yutub url_regex -i .*youtu\.be\/.*$
# logformat squid1 %{Referer}>h %ru
# access_log /var/log/squid/yt.log squid1 yutub
# acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
# acl redirec urlpath_regex -i .*&ir=1&rr=12
# cache deny redirec
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
# storeurl_access deny reddeny
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com)\/videoplayback\?(.*)/) {
$p_str = $2;
#$tag = "";
#$alg = "";
#$id = "";
#$range = "";
#if ($p_str =~ m/(itag=[0-9]*)/){$tag = "&".$1}
#if ($p_str =~ m/(algorithm=[a-z]*\-[a-z]*)/){$alg = "&".$1}
#if ($p_str =~ m/(id=[a-zA-Z0-9]*)/){$id = "&".$1}
#if ($p_str =~ m/(range=[0-9\-]*)/){$range = "&".$1; $range =~ s/-//; $range =~ s/range=//; }
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $tag . "&" . $alg . "&" . $id . "&" . $range . "\n";
#modif
@itag = m/[&?](itag=[0-9]*)/;
@CPN = m/[&?]cpn\=([a-zA-Z0-9\-\_]*)/;
@IDS = m/[&?]id\=([a-zA-Z0-9\-\_]*)/;
#@id = m/[&?](id=[^\&]*)/;
$id = &GetID($CPN[0], $IDS[0]);
@range = m/[&?](range=[^\&\s]*)/;
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/@itag&@id&@range\n";
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/id=" . $id . "&@itag@range\n";
} elsif (m/^http:\/\/www\.google-analytics\.com\/__utm\.gif\?.*/) {
print $x . "http://www.google-analytics.com/__utm.gif\n";
#Cache High Latency Ads
} elsif (m/^http:\/\/([a-z0-9.]*)(\.doubleclick\.net|\.quantserve\.com|\.googlesyndication\.com|yieldmanager|cpxinteractive)(.*)/) {
$y = $3;$z = $2;
for ($y) {
s/pixel;.*/pixel/;
s/activity;.*/activity/;
s/(imgad[^&]*).*/\1/;
s/;ord=[?0-9]*//;
s/;×tamp=[0-9]*//;
s/[&?]correlator=[0-9]*//;
s/&cookie=[^&]*//;
s/&ga_hid=[^&]*//;
s/&ga_vid=[^&]*//;
s/&ga_sid=[^&]*//;
# s/&prev_slotnames=[^&]*//
# s/&u_his=[^&]*//;
s/&dt=[^&]*//;
s/&dtd=[^&]*//;
s/&lmt=[^&]*//;
s/(&alternate_ad_url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&ref=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&cookie=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/[;&?]ord=[?0-9]*//;
s/[;&]mpvid=[^&;]*//;
s/&xpc=[^&]*//;
# yieldmanager
s/\?clickTag=[^&]*//;
s/&u=[^&]*//;
s/&slotname=[^&]*//;
s/&page_slots=[^&]*//;
}
print $x . "http://" . $1 . $2 . $y . "\n";
#cache high latency ads
} elsif (m/^http:\/\/(.*?)\/(ads)\?(.*?)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
# spicific servers starts here....
} elsif (m/^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*?)/) {
print $x . "http://" . $1 . "\n";
#cdn, varialble 1st path
} elsif (($u =~ /filehippo/) && (m/^http:\/\/(.*?)\.(.*?)\/(.*?)\/(.*)\.([a-z0-9]{3,4})(\?.*)?/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/[a-z0-9]{2,5}/cdn./;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#rapidshare
} elsif (($u =~ /rapidshare/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?)([a-z]*\.[^\/]{3}\/[a-z]*\/[0-9]*)\/(.*?)\/([^\/\?\&]{4,})$/)) {
print $x . "http://cdn." . $3 . "/SQUIDINTERNAL/" . $5 . "\n";
} elsif (($u =~ /maxporn/) && (m/^http:\/\/([^\/]*?)\/(.*?)\/([^\/]*?)(\?.*)?$/)) {
print $x . "http://" . $1 . "/SQUIDINTERNAL/" . $3 . "\n";
#like porn hub variables url and center part of the path, filename etention 3 or 4 with or without ? at the end
} elsif (($u =~ /tube8|pornhub|xvideos/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?(\.[a-z]*)?)\.([a-z]*[0-9]?\.[^\/]{3}\/[a-z]*)(.*?)((\/[a-z]*)?(\/[^\/]*){4}\.[^\/\?]{3,4})(\?.*)?$/)) {
print $x . "http://cdn." . $4 . $6 . "\n";
#...spicific servers end here.
#photos-X.ak.fbcdn.net where X a-z
} elsif (m/^http:\/\/photos-[a-z].ak.fbcdn.net\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 . "\n";
#for yimg.com video
} elsif (m/^http:\/\/(.*yimg.com)\/\/(.*)\/([^\/\?\&]*\/[^\/\?\&]*\.[^\/\?\&]{3,4})(\?.*)?$/) {
print $x . "http://cdn.yimg.com//" . $3 . "\n";
#for yimg.com doubled
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*?)\.yimg\.com\/(.*?)\?(.*)/) {
print $x . "http://cdn.yimg.com/" . $3 . "\n";
#for yimg.com with &sig=
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*)/) {
@y = ($1,$2);
$y[0] =~ s/[a-z]+[0-9]+/cdn/;
$y[1] =~ s/&sig=.*//;
print $x . "http://" . $y[0] . ".yimg.com/" . $y[1] . "\n";
#youjizz. We use only domain and filename
} elsif (($u =~ /media[0-9]{2,5}\.youjizz/) && (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\/([^\/\?\&]*)\.([^\/\?\&]{3,4})((\?|\%).*)?$/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#general purpose for cdn servers. add above your specific servers.
} elsif (m/^http:\/\/([0-9.]*?)\/\/(.*?)\.(.*)\?(.*?)/) {
print $x . "http://squid-cdn-url//" . $2 . "." . $3 . "\n";
#generic http://variable.domain.com/path/filename."ex" "ext" or "exte" with or withour "? or %"
} elsif (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\.([^\/\?\&]{2,4})((\?|\%).*)?$/) {
@y = ($1,$2,$3,$4);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
# generic http://variable.domain.com/...
} elsif (m/^http:\/\/(([A-Za-z]+[0-9-]+)*?|.*cdn.*|.*cache.*)\.(.*?)\.(.*?)\/(.*)$/) {
print $x . "http://cdn." . $3 . "." . $4 . "/" . $5 . "\n";
# spicific extention that ends with ?
} elsif (m/^http:\/\/(.*?)\/(.*?)\.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|on2)(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "." . $3 . "\n";
# all that ends with ;
} elsif (m/^http:\/\/(.*?)\/(.*?)\;(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
} else {
print $x . $_ . "sucks\n";
}
}
sub GetID
{
$id = "";
use File::ReadBackwards;
my $lim = 200 ;
my $ref_log = File::ReadBackwards->new('/var/squid/logs/yt.log');
while (defined($line = $ref_log->readline))
{
if ($line =~ m/.*youtube.*\/watch\?.*v=([a-zA-Z0-9\-\_]*).*\s.*id=$IDS[0].*/){
$id = $1;
last;
}
if ($line =~ m/.*youtube.*\/.*cpn=$CPN[0].*[&](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*/){
$id = $2;
last;
}
if ($line =~ m/.*youtube.*\/.*[&?](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*cpn=$CPN[0].*/){
$id = $2;
last;
}
last if --$lim <= 0;
}
if ($id eq ""){
$id = $IDS[0];
}
$ref_log->close();
return $id;
}
#------------------------------
POSTED BY ARDIANVIRANTO ARDIAN ON 03.010 KOMENTAR
1. Install pfsense2.0.3
http://files.nyi.pfsense.org/mirror/downloads/old/pfSense-LiveCD-2.0.3-RELEASE- i386-20130412-1022.iso.gz
2. Install lusca pada pfsense2.0.3
3. Upgrade pfsense secara online
4. Reboot.
5. Re-install lusca di Pfsense 2.1
serta cek file-file di dalam squid (include.conf, storeurl.conf, dll)
6. Konfigurasi ulang melalui webconfig-->service-->proxyserver
7. Restart lusca melalui --->status-->services-->pilih lusca
8. Harusnya Jalan bila tidak jalan ulang ke step 5.
Bila Youtube tidak jalan bisa ikuti langkah sebagai berikut :
dari konsole/putty
#pkg_add -r -v -f perl
#pkg_add -r -v -f p5-File-ReadBackwardsSelanjutnya ikuti langkah dibawah ini :
#cd /root
/ buat folder text documen copy file dibawah taruh di root dengan jdl sprti dbwh
buat file cobaReadBack.pl
isinya :
#-------------------------------
#!/usr/bin/perl
use File::ReadBackwards;
$fh = File::ReadBackwards->new('/var/squid/logs/access.log') or
die "can't read file: $!\n";
while ( defined($line = $fh->readline) )
{
print $line ;
}
#------------------------------- Attribut file
#chmod 0755 cobaReadBack.pl
cara tes atau menjalankannya :
# perl cobaReadBack.pl
-----------------------------
Perhatian.. folder log adalah : /var/squid/logs
----------------------------
Instal ccze supaya report nya berwarna :
#pkg_add -r -v -f ccze
2.Backup file /user/local/etc/squid/include.conf dan hapus semua isinya dan
paste kan teks dibawah ini :
#-----------------------
#thanks to ucok_karnadi@yahoo.com
#debug_options ALL,3
strip_query_terms off
acl yutub url_regex -i .*youtube\.com\/.*$
acl yutub url_regex -i .*youtu\.be\/.*$
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%h %ru
logformat squid1 %{Referer}>h %ru
#logformat squid1 %>h %ru
access_log /var/squid/logs/yt.log squid1 yutub
acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
acl redirec urlpath_regex -i .*&ir=1&rr=12
cache deny redirec
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
#acl range url_regex -i .*youtube\.com\/videoplayback.*range\=.*$
#http_access deny range
#storeurl_access deny reddeny
#
# $Rev$
## LUSCA
acl speedtest url_regex ^http:\/\/.*\/speedtest\/.*
acl store_rewrite_list urlpath_regex .*\.ak\.fbcdn\.net\/
acl store_rewrite_list urlpath_regex http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)
acl store_rewrite_list urlpath_regex s[0-9]*\.filesonic\.com\/download\/.*
acl store_rewrite_list urlpath_regex [a-zA-Z]{2}[0-9]*\.4shared\.com\/download\/
acl store_rewrite_list urlpath_regex \/(watch\?|get_video\?|videodownload\?|videoplayback.*id)
#acl store_rewrite_list urlpath_regex \.youtube\.com\/videoplayback \.youtube\.com\/videoplay \.youtube\.com\/get_video\?
#acl store_rewrite_list urlpath_regex \.youtube\.[a-z][a-z]\/videoplayback \.youtube\.[a-z][a-z]\/videoplay \.youtube\.[a-z][a-z]\/get_video\?
acl store_rewrite_list urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|rar|cab)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|exe|cab)$
acl store_rewrite_list_domain_CDN url_regex \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.*
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[a-z]+[0-9]\.google\.co(m|\.id)
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/\.www[0-9][0-9]\.indowebster\.com\/(.*)(rar|zip|flv|wm(a|v)|3gp|mp(4|3)|exe|msi|avi|(mp(e?g|a|e|1|2|3|4))|cab|exe)
acl store_rewrite_list urlpath_regex \/(get_video|videoplayback\?id|videoplayback.*id) \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|swf)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|krf|exe)$
acl store_rewrite_list_domain_CDN url_regex (khm|mt)[0-9]?.google.co(m|\.id) streamate.doublepimp.com.*\.js\? photos-[a-z].ak.fbcdn.net \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.* yieldmanager cpxinteractive ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl dontrewrite url_regex redbot\.org (get_video|videoplayback\?id|videoplayback.*id).*begin\=[1-9][0-9]* \.php\?
acl getmethod method GET
storeurl_access allow speedtest
storeurl_access deny dontrewrite
storeurl_access deny !getmethod
storeurl_access allow store_rewrite_list_domain_CDN
storeurl_access allow store_rewrite_list
storeurl_access allow store_rewrite_list_domain
storeurl_access allow store_rewrite_list_path
storeurl_access deny reddeny
storeurl_access deny all
storeurl_rewrite_program /usr/local/etc/squid/storeurl.pl
storeurl_rewrite_children 16
storeurl_rewrite_concurrency 99
acl snmppublic snmp_community public
cachemgr_passwd none config reconfigure
#work around for fragment videos of msn
acl msnvideo url_regex QualityLevel.*Fragment
http_access deny msnvideo
#always_direct allow html
#cache_peer localhost parent 4001 0 carp login=PASS name=backend-1
max_stale 10 years
include /usr/local/etc/squid/refresh.conf
#acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
#upgrade_http0.9 deny shoutcast
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
#read_ahead_gap 0 KB
#ie_refresh on
reload_into_ims on
strip_query_terms off
deny_info TCP_RESET localnet
negative_dns_ttl 1 second
negative_ttl 1 second
snmp_port 3401
snmp_access allow snmppublic all
maximum_single_addr_tries 2
retry_on_error on
#n_aiops_threads 64
# server_http11 on
#request_header_max_size 128 KB
#reply_header_max_size 128 KB
#range_offset_limit 10 MB
vary_ignore_expire on
#client_db off # this needs to be on for acl maxconn to work
ipcache_size 4096
fqdncache_size 20
#tcp_recv_bufsize 64 KB
pipeline_prefetch on
#half_closed_clients off
# 0x10 no delay, 0x08 throughput, 0x04 reliability
# 0x10 10000 (minimize delay) Use delay metric
# 0x08 01000 (maximize throughput) Use default metric
# 0x04 00100 (maximize reliability) Use reliability metric
# 0x02 00010 (minimize monetary cost) Use cost metric
# dscp squidtos+ECN
# 56 0xE0 11100000
# 48 0xc0 11000000
# 08 0x20 00100000
# 32 0x80 10000000
# 16 0x40 01000000
#tcp_outgoing_tos 0x03 video
#tcp_outgoing_tos 0xb8 html
#tcp_outgoing_tos 0x20 images
#tcp_outgoing_tos 0x02 all
#zph_mode tos
#zph_local 0xb8
#zph_parent 0x08
#acl monitor url_regex avira
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%
#---------------------------------
3. Backup file /user/local/etc/squid/storeurl.pl dan hapus semua isinya dan paste kan teks dibawah ini :
#---------------------------------
#!/usr/bin/perl
# store rewrite originally writen by chudy_fernandez@yahoo.com
# modified by member of comstuff.net to satisfy common and dynamic content.
#
$|=1;
while (<>) {
@X = split;
# $X[1] =~ s/&sig=.*//;
$x = $X[0] . " ";
$_ = $X[1];
$u = $X[1];
#speedtest
if (m/^http:\/\/(.*)\/speedtest\/(.*\.(jpg|txt))\?(.*)/) {
print $x . "http://www.speedtest.net.SQUIDINTERNAL/speedtest/" . $2 . "\n";
#mediafire
}elsif (m/^http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)/) {
print $x . "http://www.mediafire.com.SQUIDINTERNAL/" . $1 ."/" . $2 . "\n";
#fileserve
}elsif (m/^http:\/\/fs\w*\.fileserve\.com\/file\/(\w*)\/[\w-]*\.\/(.*)/) {
print $x . "http://www.fileserve.com.SQUIDINTERNAL/" . $1 . "./" . $2 . "\n";
#filesonic
}elsif (m/^http:\/\/s[0-9]*\.filesonic\.com\/download\/([0-9]*)\/(.*)/) {
print $x . "http://www.filesonic.com.SQUIDINTERNAL/" . $1 . "\n";
#4shared
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/download\/(.*)\/(.*\..*)\?.*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/download/$2\/$3\n";
#4shared preview
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/img\/(\d*)\/\w*\/dlink__2Fdownload_2F(\w*)_3Ftsid_3D[\w-]*\/preview\.mp3\?sId=\w*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/$2\n";
#photos-X.ak.fbcdn.net where X a-z
}elsif (m/^http:\/\/photos-[a-z](\.ak\.fbcdn\.net)(\/.*\/)(.*\.jpg)/) {
print $x . "http://photos" . $1 . "/" . $2 . $3 . "\n";
#YX.sphotos.ak.fbcdn.net where X 1-9, Y a-z
} elsif (m/^http:\/\/[a-z][0-9]\.sphotos\.ak\.fbcdn\.net\/(.*)\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 ."/". $2 . "\n";
#maps.google.com
} elsif (m/^http:\/\/(cbk|mt|khm|mlt|tbn)[0-9]?(.google\.co(m|\.uk|\.id).*)/) {
print $x . "http://" . $1 . $2 . "\n";
# compatibility for old cached get_video?video_id
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com).*?(videoplayback\?id=.*?|video_id=.*?)\&(.*?)/) {
$z = $2; $z =~ s/video_id=/get_video?video_id=/;
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $z . "\n";
# youtube fix by th30nly @comstuff.net, ucok_karnadi@yahoo.com, extreemblank@yahoo.com
# for ALL Youtube ( range & non range )
# first you need do this
# install package dependencies "apt-get install libfile-readbackwards-perl"
# add line below to your squid config and remove "#"
# strip_query_terms off
# acl yutub url_regex -i .*youtube\.com\/.*$
# acl yutub url_regex -i .*youtu\.be\/.*$
# logformat squid1 %{Referer}>h %ru
# access_log /var/log/squid/yt.log squid1 yutub
# acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
# acl redirec urlpath_regex -i .*&ir=1&rr=12
# cache deny redirec
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
# storeurl_access deny reddeny
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com)\/videoplayback\?(.*)/) {
$p_str = $2;
#$tag = "";
#$alg = "";
#$id = "";
#$range = "";
#if ($p_str =~ m/(itag=[0-9]*)/){$tag = "&".$1}
#if ($p_str =~ m/(algorithm=[a-z]*\-[a-z]*)/){$alg = "&".$1}
#if ($p_str =~ m/(id=[a-zA-Z0-9]*)/){$id = "&".$1}
#if ($p_str =~ m/(range=[0-9\-]*)/){$range = "&".$1; $range =~ s/-//; $range =~ s/range=//; }
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $tag . "&" . $alg . "&" . $id . "&" . $range . "\n";
#modif
@itag = m/[&?](itag=[0-9]*)/;
@CPN = m/[&?]cpn\=([a-zA-Z0-9\-\_]*)/;
@IDS = m/[&?]id\=([a-zA-Z0-9\-\_]*)/;
#@id = m/[&?](id=[^\&]*)/;
$id = &GetID($CPN[0], $IDS[0]);
@range = m/[&?](range=[^\&\s]*)/;
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/@itag&@id&@range\n";
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/id=" . $id . "&@itag@range\n";
} elsif (m/^http:\/\/www\.google-analytics\.com\/__utm\.gif\?.*/) {
print $x . "http://www.google-analytics.com/__utm.gif\n";
#Cache High Latency Ads
} elsif (m/^http:\/\/([a-z0-9.]*)(\.doubleclick\.net|\.quantserve\.com|\.googlesyndication\.com|yieldmanager|cpxinteractive)(.*)/) {
$y = $3;$z = $2;
for ($y) {
s/pixel;.*/pixel/;
s/activity;.*/activity/;
s/(imgad[^&]*).*/\1/;
s/;ord=[?0-9]*//;
s/;×tamp=[0-9]*//;
s/[&?]correlator=[0-9]*//;
s/&cookie=[^&]*//;
s/&ga_hid=[^&]*//;
s/&ga_vid=[^&]*//;
s/&ga_sid=[^&]*//;
# s/&prev_slotnames=[^&]*//
# s/&u_his=[^&]*//;
s/&dt=[^&]*//;
s/&dtd=[^&]*//;
s/&lmt=[^&]*//;
s/(&alternate_ad_url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&ref=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&cookie=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/[;&?]ord=[?0-9]*//;
s/[;&]mpvid=[^&;]*//;
s/&xpc=[^&]*//;
# yieldmanager
s/\?clickTag=[^&]*//;
s/&u=[^&]*//;
s/&slotname=[^&]*//;
s/&page_slots=[^&]*//;
}
print $x . "http://" . $1 . $2 . $y . "\n";
#cache high latency ads
} elsif (m/^http:\/\/(.*?)\/(ads)\?(.*?)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
# spicific servers starts here....
} elsif (m/^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*?)/) {
print $x . "http://" . $1 . "\n";
#cdn, varialble 1st path
} elsif (($u =~ /filehippo/) && (m/^http:\/\/(.*?)\.(.*?)\/(.*?)\/(.*)\.([a-z0-9]{3,4})(\?.*)?/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/[a-z0-9]{2,5}/cdn./;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#rapidshare
} elsif (($u =~ /rapidshare/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?)([a-z]*\.[^\/]{3}\/[a-z]*\/[0-9]*)\/(.*?)\/([^\/\?\&]{4,})$/)) {
print $x . "http://cdn." . $3 . "/SQUIDINTERNAL/" . $5 . "\n";
} elsif (($u =~ /maxporn/) && (m/^http:\/\/([^\/]*?)\/(.*?)\/([^\/]*?)(\?.*)?$/)) {
print $x . "http://" . $1 . "/SQUIDINTERNAL/" . $3 . "\n";
#like porn hub variables url and center part of the path, filename etention 3 or 4 with or without ? at the end
} elsif (($u =~ /tube8|pornhub|xvideos/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?(\.[a-z]*)?)\.([a-z]*[0-9]?\.[^\/]{3}\/[a-z]*)(.*?)((\/[a-z]*)?(\/[^\/]*){4}\.[^\/\?]{3,4})(\?.*)?$/)) {
print $x . "http://cdn." . $4 . $6 . "\n";
#...spicific servers end here.
#photos-X.ak.fbcdn.net where X a-z
} elsif (m/^http:\/\/photos-[a-z].ak.fbcdn.net\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 . "\n";
#for yimg.com video
} elsif (m/^http:\/\/(.*yimg.com)\/\/(.*)\/([^\/\?\&]*\/[^\/\?\&]*\.[^\/\?\&]{3,4})(\?.*)?$/) {
print $x . "http://cdn.yimg.com//" . $3 . "\n";
#for yimg.com doubled
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*?)\.yimg\.com\/(.*?)\?(.*)/) {
print $x . "http://cdn.yimg.com/" . $3 . "\n";
#for yimg.com with &sig=
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*)/) {
@y = ($1,$2);
$y[0] =~ s/[a-z]+[0-9]+/cdn/;
$y[1] =~ s/&sig=.*//;
print $x . "http://" . $y[0] . ".yimg.com/" . $y[1] . "\n";
#youjizz. We use only domain and filename
} elsif (($u =~ /media[0-9]{2,5}\.youjizz/) && (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\/([^\/\?\&]*)\.([^\/\?\&]{3,4})((\?|\%).*)?$/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#general purpose for cdn servers. add above your specific servers.
} elsif (m/^http:\/\/([0-9.]*?)\/\/(.*?)\.(.*)\?(.*?)/) {
print $x . "http://squid-cdn-url//" . $2 . "." . $3 . "\n";
#generic http://variable.domain.com/path/filename."ex" "ext" or "exte" with or withour "? or %"
} elsif (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\.([^\/\?\&]{2,4})((\?|\%).*)?$/) {
@y = ($1,$2,$3,$4);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
# generic http://variable.domain.com/...
} elsif (m/^http:\/\/(([A-Za-z]+[0-9-]+)*?|.*cdn.*|.*cache.*)\.(.*?)\.(.*?)\/(.*)$/) {
print $x . "http://cdn." . $3 . "." . $4 . "/" . $5 . "\n";
# spicific extention that ends with ?
} elsif (m/^http:\/\/(.*?)\/(.*?)\.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|on2)(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "." . $3 . "\n";
# all that ends with ;
} elsif (m/^http:\/\/(.*?)\/(.*?)\;(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
} else {
print $x . $_ . "sucks\n";
}
}
sub GetID
{
$id = "";
use File::ReadBackwards;
my $lim = 200 ;
my $ref_log = File::ReadBackwards->new('/var/squid/logs/yt.log');
while (defined($line = $ref_log->readline))
{
if ($line =~ m/.*youtube.*\/watch\?.*v=([a-zA-Z0-9\-\_]*).*\s.*id=$IDS[0].*/){
$id = $1;
last;
}
if ($line =~ m/.*youtube.*\/.*cpn=$CPN[0].*[&](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*/){
$id = $2;
last;
}
if ($line =~ m/.*youtube.*\/.*[&?](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*cpn=$CPN[0].*/){
$id = $2;
last;
}
last if --$lim <= 0;
}
if ($id eq ""){
$id = $IDS[0];
}
$ref_log->close();
return $id;
}
#------------------------------
PFsense Package For Bandwidth Monitoring - Ntop Installation Guide
PFsense Package For Bandwidth Monitoring - Ntop Installation Guide
Ntop Installation Guide:
As far as my experience is concerned. I have been using ntop for over an year now. It works really fine with reasonable statistics. Installation Guide.
1- Open Pfsense Web Interface
2-Navigate to System Tab
3-Select Packages
4-Click Available Packages
5-Search ntop
6-Click Install
And you're done.
As far as my experience is concerned. I have been using ntop for over an year now. It works really fine with reasonable statistics. Installation Guide.
1- Open Pfsense Web Interface
2-Navigate to System Tab
3-Select Packages
4-Click Available Packages
5-Search ntop
6-Click Install
And you're done.
Pfsense Proxy Servers
Proxy Servers
Proxy Server & Proxy Filter Configuration:
Squid:
Now we discuss here about the configuration of Proxy Server and Proxy Filter (Squid and Squid Guard). First of we discuss here what is “Squid”.
Squid:
This is a high performance web cache proxy server.
Now we come to the Configuration.
Go to the services tab and select Proxy Server.
Access Control: 192.168.0.0/24
Note: In access control section we can add two or more different networks. In this case proxy server allow internet access of both networks and both communicate to each other. Like as
192.168.0.0/24
192.168.1.0/24
192.168.2.0/24
Squid Guard:
This is a high performance web cache proxy server
Note: Squid installation is most important before the installation of SquidGuard. Now we go to next step Configuration.
If we need more feature like as “IP Blocking” “MAC Blocking” “Domain Filtering” “Timing schedule” “2 Different Network Communication” etc. Configuration
add one Timing Schedule by hitting on + plus sign.
Name: Weekly Timing Schedule
Description: Weekly Timing Schedule Listed here
Save Settings
Next go to the Target Categories Tab in Proxy Filter. Add one Target Categore by hitting on + plus sign.
Name: Domain_List
Domain List: facebook.com youtube.com twitter.com
Note: Every domain separated by a single space.
Save Settings
Next go to the ACL_Group tab in Proxy Filter. Add one ACL _Group by hitting on + plus sign.
Name: ACL_Group
Client (Source): 192.168.0.3-192.168.0.254
Timing: Weekly Timing Schedule
Target Rule: Hit on play sign stay on Target categories Select timing rule and Deny
Save Settings
Next go to the Common ACL Tab in Proxy Filter. Add one Common ACL by hitting on + plus sign.
Next go to the Common ACL Tab in Proxy Filter. Add one Common ACL by hitting on + plus sign.
Target Rule: Access denied
Save Settings
Save Settings
Next go to the General Setting Tab in Proxy Filter. Add one General Setting by hitting on + plus sign.
Enable: Add check sign
Apply Settings
Subscribe to:
Posts (Atom)