Lusca Patch 2014 ( Youtube Cache)
POSTED BY ARDIANVIRANTO ARDIAN ON 03.010 KOMENTAR
1. Install pfsense2.0.3
http://files.nyi.pfsense.org/mirror/downloads/old/pfSense-LiveCD-2.0.3-RELEASE- i386-20130412-1022.iso.gz
2. Install lusca pada pfsense2.0.3
3. Upgrade pfsense secara online
4. Reboot.
5. Re-install lusca di Pfsense 2.1
serta cek file-file di dalam squid (include.conf, storeurl.conf, dll)
6. Konfigurasi ulang melalui webconfig-->service-->proxyserver
7. Restart lusca melalui --->status-->services-->pilih lusca
8. Harusnya Jalan bila tidak jalan ulang ke step 5.
Bila Youtube tidak jalan bisa ikuti langkah sebagai berikut :
dari konsole/putty
#pkg_add -r -v -f perl
#pkg_add -r -v -f p5-File-ReadBackwardsSelanjutnya ikuti langkah dibawah ini :
#cd /root
/ buat folder text documen copy file dibawah taruh di root dengan jdl sprti dbwh
buat file cobaReadBack.pl
isinya :
#-------------------------------
#!/usr/bin/perl
use File::ReadBackwards;
$fh = File::ReadBackwards->new('/var/squid/logs/access.log') or
die "can't read file: $!\n";
while ( defined($line = $fh->readline) )
{
print $line ;
}
#------------------------------- Attribut file
#chmod 0755 cobaReadBack.pl
cara tes atau menjalankannya :
# perl cobaReadBack.pl
-----------------------------
Perhatian.. folder log adalah : /var/squid/logs
----------------------------
Instal ccze supaya report nya berwarna :
#pkg_add -r -v -f ccze
2.Backup file /user/local/etc/squid/include.conf dan hapus semua isinya dan
paste kan teks dibawah ini :
#-----------------------
#thanks to ucok_karnadi@yahoo.com
#debug_options ALL,3
strip_query_terms off
acl yutub url_regex -i .*youtube\.com\/.*$
acl yutub url_regex -i .*youtu\.be\/.*$
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%h %ru
logformat squid1 %{Referer}>h %ru
#logformat squid1 %>h %ru
access_log /var/squid/logs/yt.log squid1 yutub
acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
acl redirec urlpath_regex -i .*&ir=1&rr=12
cache deny redirec
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
#acl range url_regex -i .*youtube\.com\/videoplayback.*range\=.*$
#http_access deny range
#storeurl_access deny reddeny
#
# $Rev$
## LUSCA
acl speedtest url_regex ^http:\/\/.*\/speedtest\/.*
acl store_rewrite_list urlpath_regex .*\.ak\.fbcdn\.net\/
acl store_rewrite_list urlpath_regex http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)
acl store_rewrite_list urlpath_regex s[0-9]*\.filesonic\.com\/download\/.*
acl store_rewrite_list urlpath_regex [a-zA-Z]{2}[0-9]*\.4shared\.com\/download\/
acl store_rewrite_list urlpath_regex \/(watch\?|get_video\?|videodownload\?|videoplayback.*id)
#acl store_rewrite_list urlpath_regex \.youtube\.com\/videoplayback \.youtube\.com\/videoplay \.youtube\.com\/get_video\?
#acl store_rewrite_list urlpath_regex \.youtube\.[a-z][a-z]\/videoplayback \.youtube\.[a-z][a-z]\/videoplay \.youtube\.[a-z][a-z]\/get_video\?
acl store_rewrite_list urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|rar|cab)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|exe|cab)$
acl store_rewrite_list_domain_CDN url_regex \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.*
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/[a-z]+[0-9]\.google\.co(m|\.id)
acl store_rewrite_list_domain_CDN url_regex ^http:\/\/\.www[0-9][0-9]\.indowebster\.com\/(.*)(rar|zip|flv|wm(a|v)|3gp|mp(4|3)|exe|msi|avi|(mp(e?g|a|e|1|2|3|4))|cab|exe)
acl store_rewrite_list urlpath_regex \/(get_video|videoplayback\?id|videoplayback.*id) \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|wmv|3gp|mp(4|3)|exe|msi|zip|on2|mar|swf)\?
acl store_rewrite_list_domain url_regex ^http:\/\/([a-zA-Z-]+[0-9-]+)\.[A-Za-z]*\.[A-Za-z]*
acl store_rewrite_list_domain url_regex (([a-z]{1,2}[0-9]{1,3})|([0-9]{1,3}[a-z]{1,2}))\.[a-z]*[0-9]?\.[a-z]{3}
acl store_rewrite_list_path urlpath_regex \.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|avc|zip|mp3|3gp|rar|on2|mar|krf|exe)$
acl store_rewrite_list_domain_CDN url_regex (khm|mt)[0-9]?.google.co(m|\.id) streamate.doublepimp.com.*\.js\? photos-[a-z].ak.fbcdn.net \.rapidshare\.com.*\/[0-9]*\/.*\/[^\/]* ^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*) \.doubleclick\.net.* yieldmanager cpxinteractive ^http:\/\/[.a-z0-9]*\.photobucket\.com.*\.[a-z]{3}$ quantserve\.com
acl dontrewrite url_regex redbot\.org (get_video|videoplayback\?id|videoplayback.*id).*begin\=[1-9][0-9]* \.php\?
acl getmethod method GET
storeurl_access allow speedtest
storeurl_access deny dontrewrite
storeurl_access deny !getmethod
storeurl_access allow store_rewrite_list_domain_CDN
storeurl_access allow store_rewrite_list
storeurl_access allow store_rewrite_list_domain
storeurl_access allow store_rewrite_list_path
storeurl_access deny reddeny
storeurl_access deny all
storeurl_rewrite_program /usr/local/etc/squid/storeurl.pl
storeurl_rewrite_children 16
storeurl_rewrite_concurrency 99
acl snmppublic snmp_community public
cachemgr_passwd none config reconfigure
#work around for fragment videos of msn
acl msnvideo url_regex QualityLevel.*Fragment
http_access deny msnvideo
#always_direct allow html
#cache_peer localhost parent 4001 0 carp login=PASS name=backend-1
max_stale 10 years
include /usr/local/etc/squid/refresh.conf
#acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
#upgrade_http0.9 deny shoutcast
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
#read_ahead_gap 0 KB
#ie_refresh on
reload_into_ims on
strip_query_terms off
deny_info TCP_RESET localnet
negative_dns_ttl 1 second
negative_ttl 1 second
snmp_port 3401
snmp_access allow snmppublic all
maximum_single_addr_tries 2
retry_on_error on
#n_aiops_threads 64
# server_http11 on
#request_header_max_size 128 KB
#reply_header_max_size 128 KB
#range_offset_limit 10 MB
vary_ignore_expire on
#client_db off # this needs to be on for acl maxconn to work
ipcache_size 4096
fqdncache_size 20
#tcp_recv_bufsize 64 KB
pipeline_prefetch on
#half_closed_clients off
# 0x10 no delay, 0x08 throughput, 0x04 reliability
# 0x10 10000 (minimize delay) Use delay metric
# 0x08 01000 (maximize throughput) Use default metric
# 0x04 00100 (maximize reliability) Use reliability metric
# 0x02 00010 (minimize monetary cost) Use cost metric
# dscp squidtos+ECN
# 56 0xE0 11100000
# 48 0xc0 11000000
# 08 0x20 00100000
# 32 0x80 10000000
# 16 0x40 01000000
#tcp_outgoing_tos 0x03 video
#tcp_outgoing_tos 0xb8 html
#tcp_outgoing_tos 0x20 images
#tcp_outgoing_tos 0x02 all
#zph_mode tos
#zph_local 0xb8
#zph_parent 0x08
#acl monitor url_regex avira
#logformat chudy %ts.%03tu %6tr %>a %Ss/%03Hs %h] Rp[%
#---------------------------------
3. Backup file /user/local/etc/squid/storeurl.pl dan hapus semua isinya dan paste kan teks dibawah ini :
#---------------------------------
#!/usr/bin/perl
# store rewrite originally writen by chudy_fernandez@yahoo.com
# modified by member of comstuff.net to satisfy common and dynamic content.
#
$|=1;
while (<>) {
@X = split;
# $X[1] =~ s/&sig=.*//;
$x = $X[0] . " ";
$_ = $X[1];
$u = $X[1];
#speedtest
if (m/^http:\/\/(.*)\/speedtest\/(.*\.(jpg|txt))\?(.*)/) {
print $x . "http://www.speedtest.net.SQUIDINTERNAL/speedtest/" . $2 . "\n";
#mediafire
}elsif (m/^http:\/\/199\.91\.15\d\.\d*\/\w{12}\/(\w*)\/(.*)/) {
print $x . "http://www.mediafire.com.SQUIDINTERNAL/" . $1 ."/" . $2 . "\n";
#fileserve
}elsif (m/^http:\/\/fs\w*\.fileserve\.com\/file\/(\w*)\/[\w-]*\.\/(.*)/) {
print $x . "http://www.fileserve.com.SQUIDINTERNAL/" . $1 . "./" . $2 . "\n";
#filesonic
}elsif (m/^http:\/\/s[0-9]*\.filesonic\.com\/download\/([0-9]*)\/(.*)/) {
print $x . "http://www.filesonic.com.SQUIDINTERNAL/" . $1 . "\n";
#4shared
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/download\/(.*)\/(.*\..*)\?.*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/download/$2\/$3\n";
#4shared preview
}elsif (m/^http:\/\/[a-zA-Z]{2}\d*\.4shared\.com(:8080|)\/img\/(\d*)\/\w*\/dlink__2Fdownload_2F(\w*)_3Ftsid_3D[\w-]*\/preview\.mp3\?sId=\w*/) {
print $x . "http://www.4shared.com.SQUIDINTERNAL/$2\n";
#photos-X.ak.fbcdn.net where X a-z
}elsif (m/^http:\/\/photos-[a-z](\.ak\.fbcdn\.net)(\/.*\/)(.*\.jpg)/) {
print $x . "http://photos" . $1 . "/" . $2 . $3 . "\n";
#YX.sphotos.ak.fbcdn.net where X 1-9, Y a-z
} elsif (m/^http:\/\/[a-z][0-9]\.sphotos\.ak\.fbcdn\.net\/(.*)\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 ."/". $2 . "\n";
#maps.google.com
} elsif (m/^http:\/\/(cbk|mt|khm|mlt|tbn)[0-9]?(.google\.co(m|\.uk|\.id).*)/) {
print $x . "http://" . $1 . $2 . "\n";
# compatibility for old cached get_video?video_id
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com).*?(videoplayback\?id=.*?|video_id=.*?)\&(.*?)/) {
$z = $2; $z =~ s/video_id=/get_video?video_id=/;
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $z . "\n";
# youtube fix by th30nly @comstuff.net, ucok_karnadi@yahoo.com, extreemblank@yahoo.com
# for ALL Youtube ( range & non range )
# first you need do this
# install package dependencies "apt-get install libfile-readbackwards-perl"
# add line below to your squid config and remove "#"
# strip_query_terms off
# acl yutub url_regex -i .*youtube\.com\/.*$
# acl yutub url_regex -i .*youtu\.be\/.*$
# logformat squid1 %{Referer}>h %ru
# access_log /var/log/squid/yt.log squid1 yutub
# acl redirec urlpath_regex -i .*&redirect_counter=1&cms_redirect=yes
# acl redirec urlpath_regex -i .*&ir=1&rr=12
# cache deny redirec
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*redirect_counter=1.*$
# acl reddeny url_regex -i c\.youtube\.com\/videoplayback.*cms_redirect=yes.*$
# storeurl_access deny reddeny
} elsif (m/^http:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com)\/videoplayback\?(.*)/) {
$p_str = $2;
#$tag = "";
#$alg = "";
#$id = "";
#$range = "";
#if ($p_str =~ m/(itag=[0-9]*)/){$tag = "&".$1}
#if ($p_str =~ m/(algorithm=[a-z]*\-[a-z]*)/){$alg = "&".$1}
#if ($p_str =~ m/(id=[a-zA-Z0-9]*)/){$id = "&".$1}
#if ($p_str =~ m/(range=[0-9\-]*)/){$range = "&".$1; $range =~ s/-//; $range =~ s/range=//; }
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/" . $tag . "&" . $alg . "&" . $id . "&" . $range . "\n";
#modif
@itag = m/[&?](itag=[0-9]*)/;
@CPN = m/[&?]cpn\=([a-zA-Z0-9\-\_]*)/;
@IDS = m/[&?]id\=([a-zA-Z0-9\-\_]*)/;
#@id = m/[&?](id=[^\&]*)/;
$id = &GetID($CPN[0], $IDS[0]);
@range = m/[&?](range=[^\&\s]*)/;
#print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/@itag&@id&@range\n";
print $x . "http://video-srv.youtube.com.SQUIDINTERNAL/id=" . $id . "&@itag@range\n";
} elsif (m/^http:\/\/www\.google-analytics\.com\/__utm\.gif\?.*/) {
print $x . "http://www.google-analytics.com/__utm.gif\n";
#Cache High Latency Ads
} elsif (m/^http:\/\/([a-z0-9.]*)(\.doubleclick\.net|\.quantserve\.com|\.googlesyndication\.com|yieldmanager|cpxinteractive)(.*)/) {
$y = $3;$z = $2;
for ($y) {
s/pixel;.*/pixel/;
s/activity;.*/activity/;
s/(imgad[^&]*).*/\1/;
s/;ord=[?0-9]*//;
s/;×tamp=[0-9]*//;
s/[&?]correlator=[0-9]*//;
s/&cookie=[^&]*//;
s/&ga_hid=[^&]*//;
s/&ga_vid=[^&]*//;
s/&ga_sid=[^&]*//;
# s/&prev_slotnames=[^&]*//
# s/&u_his=[^&]*//;
s/&dt=[^&]*//;
s/&dtd=[^&]*//;
s/&lmt=[^&]*//;
s/(&alternate_ad_url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&url=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&ref=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/(&cookie=http%3A%2F%2F[^(%2F)]*)[^&]*/\1/;
s/[;&?]ord=[?0-9]*//;
s/[;&]mpvid=[^&;]*//;
s/&xpc=[^&]*//;
# yieldmanager
s/\?clickTag=[^&]*//;
s/&u=[^&]*//;
s/&slotname=[^&]*//;
s/&page_slots=[^&]*//;
}
print $x . "http://" . $1 . $2 . $y . "\n";
#cache high latency ads
} elsif (m/^http:\/\/(.*?)\/(ads)\?(.*?)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
# spicific servers starts here....
} elsif (m/^http:\/\/(www\.ziddu\.com.*\.[^\/]{3,4})\/(.*?)/) {
print $x . "http://" . $1 . "\n";
#cdn, varialble 1st path
} elsif (($u =~ /filehippo/) && (m/^http:\/\/(.*?)\.(.*?)\/(.*?)\/(.*)\.([a-z0-9]{3,4})(\?.*)?/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/[a-z0-9]{2,5}/cdn./;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#rapidshare
} elsif (($u =~ /rapidshare/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?)([a-z]*\.[^\/]{3}\/[a-z]*\/[0-9]*)\/(.*?)\/([^\/\?\&]{4,})$/)) {
print $x . "http://cdn." . $3 . "/SQUIDINTERNAL/" . $5 . "\n";
} elsif (($u =~ /maxporn/) && (m/^http:\/\/([^\/]*?)\/(.*?)\/([^\/]*?)(\?.*)?$/)) {
print $x . "http://" . $1 . "/SQUIDINTERNAL/" . $3 . "\n";
#like porn hub variables url and center part of the path, filename etention 3 or 4 with or without ? at the end
} elsif (($u =~ /tube8|pornhub|xvideos/) && (m/^http:\/\/(([A-Za-z]+[0-9-.]+)*?(\.[a-z]*)?)\.([a-z]*[0-9]?\.[^\/]{3}\/[a-z]*)(.*?)((\/[a-z]*)?(\/[^\/]*){4}\.[^\/\?]{3,4})(\?.*)?$/)) {
print $x . "http://cdn." . $4 . $6 . "\n";
#...spicific servers end here.
#photos-X.ak.fbcdn.net where X a-z
} elsif (m/^http:\/\/photos-[a-z].ak.fbcdn.net\/(.*)/) {
print $x . "http://photos.ak.fbcdn.net/" . $1 . "\n";
#for yimg.com video
} elsif (m/^http:\/\/(.*yimg.com)\/\/(.*)\/([^\/\?\&]*\/[^\/\?\&]*\.[^\/\?\&]{3,4})(\?.*)?$/) {
print $x . "http://cdn.yimg.com//" . $3 . "\n";
#for yimg.com doubled
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*?)\.yimg\.com\/(.*?)\?(.*)/) {
print $x . "http://cdn.yimg.com/" . $3 . "\n";
#for yimg.com with &sig=
} elsif (m/^http:\/\/(.*?)\.yimg\.com\/(.*)/) {
@y = ($1,$2);
$y[0] =~ s/[a-z]+[0-9]+/cdn/;
$y[1] =~ s/&sig=.*//;
print $x . "http://" . $y[0] . ".yimg.com/" . $y[1] . "\n";
#youjizz. We use only domain and filename
} elsif (($u =~ /media[0-9]{2,5}\.youjizz/) && (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\/([^\/\?\&]*)\.([^\/\?\&]{3,4})((\?|\%).*)?$/)) {
@y = ($1,$2,$4,$5);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
#general purpose for cdn servers. add above your specific servers.
} elsif (m/^http:\/\/([0-9.]*?)\/\/(.*?)\.(.*)\?(.*?)/) {
print $x . "http://squid-cdn-url//" . $2 . "." . $3 . "\n";
#generic http://variable.domain.com/path/filename."ex" "ext" or "exte" with or withour "? or %"
} elsif (m/^http:\/\/(.*)(\.[^\.\-]*?\..*?)\/(.*)\.([^\/\?\&]{2,4})((\?|\%).*)?$/) {
@y = ($1,$2,$3,$4);
$y[0] =~ s/(([a-zA-A]+[0-9]+(-[a-zA-Z])?$)|(.*cdn.*)|(.*cache.*))/cdn/;
print $x . "http://" . $y[0] . $y[1] . "/" . $y[2] . "." . $y[3] . "\n";
# generic http://variable.domain.com/...
} elsif (m/^http:\/\/(([A-Za-z]+[0-9-]+)*?|.*cdn.*|.*cache.*)\.(.*?)\.(.*?)\/(.*)$/) {
print $x . "http://cdn." . $3 . "." . $4 . "/" . $5 . "\n";
# spicific extention that ends with ?
} elsif (m/^http:\/\/(.*?)\/(.*?)\.(jp(e?g|e|2)|gif|png|tiff?|bmp|ico|flv|on2)(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "." . $3 . "\n";
# all that ends with ;
} elsif (m/^http:\/\/(.*?)\/(.*?)\;(.*)/) {
print $x . "http://" . $1 . "/" . $2 . "\n";
} else {
print $x . $_ . "sucks\n";
}
}
sub GetID
{
$id = "";
use File::ReadBackwards;
my $lim = 200 ;
my $ref_log = File::ReadBackwards->new('/var/squid/logs/yt.log');
while (defined($line = $ref_log->readline))
{
if ($line =~ m/.*youtube.*\/watch\?.*v=([a-zA-Z0-9\-\_]*).*\s.*id=$IDS[0].*/){
$id = $1;
last;
}
if ($line =~ m/.*youtube.*\/.*cpn=$CPN[0].*[&](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*/){
$id = $2;
last;
}
if ($line =~ m/.*youtube.*\/.*[&?](video_id|docid|v)=([a-zA-Z0-9\-\_]*).*cpn=$CPN[0].*/){
$id = $2;
last;
}
last if --$lim <= 0;
}
if ($id eq ""){
$id = $IDS[0];
}
$ref_log->close();
return $id;
}
#------------------------------
No comments:
Post a Comment