Video Cache server for Squid cache Server by Babar Zahoor

Video Cache Server Using Squid Apache and Python.

Purpose of this server is to cache all the websites like youtube.com blip.tv etc at our local squid cache server.




1. Requires
===========
a. squid >= 2.6
b. python >= 2.4
c. python-urlgrabber
d. python-iniparse
e. Apache (httpd) or any other Web Server


Note: For these steps your machine must be conneted with Internet & Preinstalled CentOS 5.3 or 5.4

Please follow the steps


[root@Squid ~]# yum install squid httpd python python-urlgrabber python-iniparse



Download the Package of videocache from http://cachevideos.com/download/ .

[root@Squid Desktop]# tar -xzvf videocache-1.9.1.tar.gz
videocache-1.9.1/
videocache-1.9.1/videocache-httpd.conf
videocache-1.9.1/scripts/
videocache-1.9.1/scripts/vccleaner
videocache-1.9.1/videocache.spec
videocache-1.9.1/setup.py
videocache-1.9.1/COPYING
videocache-1.9.1/Readme
videocache-1.9.1/videocache-sysconfig.conf
videocache-1.9.1/INSTALL
videocache-1.9.1/CHANGELOG
videocache-1.9.1/videocache.8.gz
videocache-1.9.1/videocache/
videocache-1.9.1/videocache/__init__.py
videocache-1.9.1/videocache/parser.py
videocache-1.9.1/videocache/config.py
videocache-1.9.1/videocache/Errors.py
videocache-1.9.1/videocache/videocache.py
videocache-1.9.1/update-vc
videocache-1.9.1/videocache.8.t2t
[root@Squid Desktop]#
[root@Squid Desktop]# cd videocache-1.9.1
[root@Squid videocache-1.9.1]#
[root@Squid videocache-1.9.1]# cp videocache-httpd.conf /etc/httpd/conf.d/videocache.conf
[root@Squid videocache-1.9.1]# cp videocache-sysconfig.conf /etc/videocache.conf
[root@Squid videocache-1.9.1]# cp update-vc /usr/sbin/update-vc
[root@Squid videocache-1.9.1]# chmod 744 /usr/sbin/update-vc
[root@Squid videocache-1.9.1]# cp vccleaner /usr/sbin/vccleaner
[root@Squid videocache-1.9.1]# chmod 744 /usr/sbin/vccleaner
[root@Squid videocache-1.9.1]# cp -r videocache /usr/share/
[root@Squid root]# cd /var/spool/
[root@Squid spool]# mkdir videocache
[root@Squid spool]# mkdir videocache/tmp videocache/youtube videocache/metacafe videocache/dailymotion videocache/google

videocache/vimeo videocache/wrzuta videocache/soapbox videocache/tube8 videocache/tvuol videocache/bliptv videocache/break
[root@Squid spool]# chown -R squid:squid videocache
[root@Squid spool]# chmod -R 755 videocache
[root@Squid ~]# mkdir -p /var/log/videocache
[root@Squid ~]# chmod -R 755 /var/log/videocache/
[root@Squid ~]# chown -R squid:squid /var/log/videocache/



Note: Please add these lines at the last of squid.conf file if it shows you some errors please copy prper and paste again it a


root@Squid videocache-1.9.1]#Vim /etc/squid/squid.conf

# --BEGIN-- videocache config for squid
url_rewrite_program /usr/bin/python /usr/share/videocache/videocache.py
url_rewrite_children 7
acl videocache_allow_url url_regex -i \.youtube\.com\/get_video\?
acl videocache_allow_url url_regex -i \.googlevideo\.com\/videoplayback \.googlevideo\.com\/videoplay \.googlevideo\.com\/get_video\?
acl videocache_allow_url url_regex -i \.google\.com\/videoplayback \.google\.com\/videoplay \.google\.com\/get_video\?
acl videocache_allow_url url_regex -i \.google\.[a-z][a-z]\/videoplayback \.google\.[a-z][a-z]\/videoplay \.google\.[a-z][a-z]\/get_video\?
acl videocache_allow_url url_regex -i

(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?

)\/videoplayback\?
acl videocache_allow_url url_regex -i

(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?

)\/videoplay\?
acl videocache_allow_url url_regex -i

(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?

)\/get_video\?
acl videocache_allow_url url_regex -i proxy[a-z0-9\-][a-z0-9][a-z0-9][a-z0-9]?\.dailymotion\.com\/
acl videocache_allow_url url_regex -i vid\.akm\.dailymotion\.com\/
acl videocache_allow_url url_regex -i bitcast\.vimeo\.com\/vimeo\/videos\/
acl videocache_allow_url url_regex -i va\.wrzuta\.pl\/wa[0-9][0-9][0-9][0-9]?
acl videocache_allow_url url_regex -i \.msn\.com\.edgesuite\.net\/(.*)\.flv
acl videocache_allow_url url_regex -i media[a-z0-9]?[a-z0-9]?[a-z0-9]?\.tube8\.com\/ mobile[a-z0-9]?[a-z0-9]?[a-z0-9]?\.tube8\.com\/
acl videocache_allow_url url_regex -i \.mais\.uol\.com\.br\/(.*)\.flv
acl videocache_allow_url url_regex -i \.video[a-z0-9]?[a-z0-9]?\.blip\.tv\/(.*)\.(flv|avi|mov|mp3|m4v|mp4|wmv|rm|ram)
acl videocache_allow_url url_regex -i video\.break\.com\/(.*)\.(flv|mp4)
acl videocache_deny_url url_regex -i http:\/\/[a-z][a-z]\.youtube\.com http:\/\/www\.youtube\.com
url_rewrite_access deny videocache_deny_url
url_rewrite_access allow videocache_allow_url
url_rewrite_access allow videocache_allow_dom
redirector_bypass on
# --END-- videocache config for squid

[root@Squid root]# service httpd start
[root@Squid root]# chkconfig httpd --level 35 on
[root@Squid root]# chkconfig squid --level 35 on
[root@Squid root]# service squid reload


Last step then start using it. Please open the file videocache.conf in the /etc dir then place your squid machines ip there.



[root@Squid videocache-1.9.1]# cd /etc/
[root@Squid etc]# vim /etc/videocache.conf

[main]
# file : /etc/videocache.conf


######### Please put here your cache machine IP ########

cache_host = 192.168.1.254


######## please put here your squid machines IP & Port .i.e 3128 or 8080 which you are using .

proxy = http://192.168.1.254:3128/



save & exit.


then



restart squid and apache services.



Here we done it.

start browsing from youtube.com or blip.tv then you can see your machine will create cache.
We can check the service is running with this command.


[root@Squid etc]# ps -aux | grep squid
Warning: bad syntax, perhaps a bogus '-'? See /usr/share/doc/procps-3.2.7/FAQ
root 3652 0.0 0.0 49868 1628 ? Ss 10:06 0:00 squid -D
squid 3654 0.0 0.5 57884 11392 ? S 10:06 0:02 (squid) -D
squid 3656 0.0 0.4 181916 10080 ? Ssl 10:06 0:02 (python) /usr/share/videocache/videocache.py
squid 3657 0.0 0.4 151184 9928 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3658 0.0 0.4 151184 9932 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3659 0.0 0.4 151184 9932 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3660 0.0 0.4 151184 9928 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3661 0.0 0.4 151184 9928 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3662 0.0 0.4 151184 9928 ? Ss 10:06 0:00 (python) /usr/share/videocache/videocache.py
squid 3671 0.0 0.0 3640 252 ? Ss 10:06 0:00 (unlinkd)
root 4623 0.0 0.0 58956 528 pts/6 S+ 10:23 0:00 tail -f /var/log/squid/access.log
root 14153 0.0 0.0 61196 740 pts/8 S+ 12:43 0:00 grep squid
[root@Squid etc]#

We can check the log file of Video Cache
[root@Squid etc]# tail -f /var/log/videocache/videocache.log
2009-11-02 18:42:39,624 6257 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,624 6264 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,624 6262 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,624 6260 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,626 6263 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,627 6265 - - RELOAD - videocache plugin was reloaded.
2009-11-02 18:42:39,627 6261 - - RELOAD - videocache plugin was reloaded.
2009-11-03 10:06:57,731 3656 - - XMLRPCSERVER - Starting XMLRPCServer on port 9100.
2009-11-03 10:06:57,731 3656 - - SCHEDULEDER - Download Scheduler starting.
2009-11-03 10:06:59,987 3682 - - UPDATE_SIZE - Size of all caching directories updated successfully.





We can check the log of your apache server for who is accessing your videocache


[root@Squid etc]# tail -f /var/log/httpd/access_log
192.168.1.254 - - [02/Nov/2009:17:49:19 +0500] "GET /favicon.ico HTTP/1.0" 404 287 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.4)

Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"
192.168.1.254 - - [03/Nov/2009:10:17:32 +0500] "GET /sarg HTTP/1.1" 403 280 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5) Gecko/2008121911

CentOS/3.0.5-1.el5.centos Firefox/3.0.5"
192.168.1.254 - - [03/Nov/2009:10:17:32 +0500] "GET /favicon.ico HTTP/1.1" 404 283 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5)

Gecko/2008121911 CentOS/3.0.5-1.el5.centos Firefox/3.0.5"
192.168.1.254 - - [03/Nov/2009:10:17:35 +0500] "GET /favicon.ico HTTP/1.1" 404 283 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5)

Gecko/2008121911 CentOS/3.0.5-1.el5.centos Firefox/3.0.5"
192.168.1.254 - - [03/Nov/2009:10:19:40 +0500] "GET /sarg HTTP/1.1" 403 280 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5) Gecko/2008121911

CentOS/3.0.5-1.el5.centos Firefox/3.0.5"
192.168.1.254 - - [03/Nov/2009:10:24:28 +0500] "GET /favicon.ico HTTP/1.0" 404 287 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.4)

Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"
192.168.1.254 - - [03/Nov/2009:10:31:41 +0500] "GET /videocache/ HTTP/1.0" 200 3408 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.4)

Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"
192.168.1.254 - - [03/Nov/2009:10:31:41 +0500] "GET /icons/folder.gif HTTP/1.0" 304 - "http://192.168.1.254/videocache/" "Mozilla/5.0 (Windows; U; Windows NT

5.1; en-US; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"
192.168.1.254 - - [03/Nov/2009:10:31:41 +0500] "GET /icons/blank.gif HTTP/1.0" 304 - "http://192.168.1.254/videocache/" "Mozilla/5.0 (Windows; U; Windows NT

5.1; en-US; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"
192.168.1.254 - - [03/Nov/2009:10:31:41 +0500] "GET /icons/back.gif HTTP/1.0" 304 - "http://192.168.1.254/videocache/" "Mozilla/5.0 (Windows; U; Windows NT

5.1; en-US; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"


we can check aur size of videocache is increasing

[root@Squid etc]# watch du -sh /var/spool/videocache/