squid.comf example

acl GET method GET

acl rewritedoms dstdomain .googlevideo.com

store_id_program /opt/bin/ytgv-storeid.rb
store_id_children 40 startup=10 idle=5 concurrency=0
store_id_access allow rewritedoms GET
store_id_access deny all

acl youtube dstdomain www.youtube.com
acl ytwatch urlpath_regex watch\?
icap_enable on

icap_service service_req reqmod_precache icap://127.0.0.1:2344/yt2upn/ bypass=on
adaptation_access service_req allow youtube GET ytwatch
adaptation_access service_req deny all

Youtube Google video url Prediction assitended Helper Example

   1 #!/usr/bin/ruby
   2 # encoding: utf-8
   3 
   4 require "rubygems"
   5 require "net/http"
   6 require "open-uri"
   7 require 'timeout'
   8 require "redis"
   9 require 'syslog'
  10 
  11 class Cache
  12         def initialize
  13         @host = "127.0.0.1"
  14         @db = "0"
  15         @port = 6379
  16         @redis = Redis.new(:host => @host, :port => @port)
  17         @redis.select @db
  18         end
  19 
  20         def setvid(url,vid)
  21            #return @redis.setex  "md5(" + vid+ ")",1200 ,url
  22            return true;
  23         end
  24 
  25         def geturl(vid)
  26            return @redis.get "md5(" + vid + ")"
  27         end
  28 
  29         def getytidfromgvid(vid)
  30            id = @redis.get(vid)
  31            log("Google video check getytidfromgvid[#{vid}] result [#{id}].") if $debug
  32            return id
  33         end
  34 
  35         def sfdlid(url)
  36                         m = url.match(/^http:\/\/.*\.dl\.sourceforge\.net\/(.*)/)
  37                         if m[1]
  38                                 return m[1]
  39                         else
  40                                 return nil
  41                         end
  42         end
  43 
  44         def ytimg(url)
  45                 m = url.match(/.*\.ytimg.com\/(.*\.jpg|.*\.gif|.*\.js)/)
  46                 if m[1]
  47                         return m[1]
  48                 else
  49                         return nil
  50                 end
  51         end
  52 
  53         def googlevideoYouTubeID(url)
  54                 log("Google video check [#{url}].") if $debug
  55 
  56                 id = getgvid(url)
  57                 log("Google video check getgvid[#{id}].") if $debug
  58 
  59                 itag = getgvitag(url)
  60                 range = getgvrange(url)
  61                 if id == nil
  62                         return nil
  63                 else
  64                   vid = getytidfromgvid(id[3..-1])
  65                   return nil if vid == nil or vid.size < 5
  66                   vid = "id="+vid
  67                 end
  68                 if itag != nil
  69                         vid = vid + "&" + itag
  70                 end
  71                 if range != nil
  72                         vid = vid + "&" + range
  73                 end
  74 
  75                 return vid
  76         end
  77 
  78         private
  79                 def getgvid(url)
  80                         m = url.match(/(id\=[a-zA-Z0-9\-\_\%]+)/)
  81                         return m.to_s if m != nil
  82                 end
  83 
  84                 def getgvitag(url)
  85                         m = url.match(/(itag\=[0-9\-\_]+)/)
  86                         return m.to_s if m != nil
  87                 end
  88 
  89                 def getgvrange(url)
  90                         m = url.match(/(range\=[0-9\-]+)/)
  91                         return m.to_s if m != nil
  92                 end
  93 
  94                 def getgvredirect(url)
  95                         m = url.match(/(redirect\=)([a-zA-Z0-9\-\_]+)/)
  96                         return (m.to_s + Time.now.to_i.to_s) if m != nil
  97                 end
  98 end
  99 
 100 def rewriter(request)
 101                 case request
 102                 when /^https?:\/\/[a-zA-Z0-9\-\_\.]+\.googlevideo\.com\/videoplayback\?/
 103                   log("Google video Match [#{request}].") if $debug
 104                   vid = $cache.googlevideoYouTubeID(request)
 105                   log("Google video Match VID [#{vid}].") if $debug
 106                   url = "http://ytgv.squid.internal/" + vid if vid != nil
 107                   return url
 108                 when /^http:\/\/[a-zA-Z0-9\-\_\.]+\.dl\.sourceforge\.net\/.*/
 109                   vid = $cache.sfdlid(request)
 110                   $cache.setvid(request, "http://dl.sourceforge.net.squid.internal/" + vid) if vid != nil
 111                   url = "http://dl.sourceforge.net.squid.internal/" + vid if vid != nil
 112                   return url
 113                 when /^http:\/\/[a-zA-Z0-9\-\_\.]+\.ytimg\.com\/(.*\.jpg|.*\.gif|.*\.js)/
 114                   vid = $cache.ytimg(request)
 115                   $cache.setvid(request, "http://ytimg.squid.internal/" + vid) if vid != nil
 116                   url = "http://ytimg.squid.internal/" + vid if vid != nil
 117                   return url
 118                 when /^quit.*/
 119                   exit 0
 120                 else
 121                  return ""
 122                 end
 123 end
 124 
 125 def log(msg)
 126  Syslog.log(Syslog::LOG_ERR, "%s", msg)
 127 end
 128 
 129 def eval
 130         request = gets
 131         if (request && (request.match /^[0-9]+\ /))
 132          conc(request)
 133          return true
 134         else
 135          noconc(request)
 136          return false
 137         end
 138 
 139 end
 140 
 141 def conc(request)
 142                 return if !request
 143                 request = request.split
 144                 if request[0] && request[1]
 145                         log("original request [#{request.join(" ")}].") if $debug
 146                         result = rewriter(request[1])
 147                         if result && (result.size > 10)
 148                           log("result for request [#{request.join(" ")}], [#{result}]") if $debug
 149                           url = request[0] +" OK store-id=" + result
 150                         else
 151                           url = request[0] +" ERR"
 152                         end
 153                         log("modified response [#{url}].") if $debug
 154                         puts url
 155                 else
 156                         log("original request [had a problem].") if $debug
 157                         url = request[0] + "ERR"
 158                         log("modified response [#{url}].") if $debug
 159                         puts url
 160                 end
 161 
 162 end
 163 
 164 def noconc(request)
 165                 return if !request
 166                 request = request.split
 167                 if request[0]
 168                         log("Original request [#{request.join(" ")}].") if $debug
 169                         result = rewriter(request[0])
 170                         log("result for request [#{request.join(" ")}], [#{result}]") if $debug
 171                         if result && (result.size > 10)
 172                                 url = "OK store-id=" + rewriter(request[0])
 173                         else
 174                                 url = "ERR"
 175                         end
 176                         log("modified response [#{url}].") if $debug
 177                         puts url
 178                 else
 179                         log("Original request [had a problem].") if $debug
 180                         url = "ERR"
 181                         log("modified response [#{url}].") if $debug
 182                         puts url
 183                 end
 184 end
 185 
 186 def validr?(request)
 187   if (request.ascii_only? && request.valid_encoding?)
 188     return true
 189   else
 190     STDERR.puts("errorness line#{request}")
 191     return false
 192   end
 193 
 194 
 195 end
 196 
 197 def main
 198 
 199         Syslog.open('ytgv.rb', Syslog::LOG_PID)
 200         log("Started")
 201 
 202         c = eval
 203 
 204          if c
 205           while request = gets
 206              conc(request) if validr?(request)
 207           end
 208          else
 209           while request = gets
 210              noconc(request) if validr?(request)
 211           end
 212          end
 213 end
 214 
 215 $debug = true
 216 $cache = Cache.new
 217 STDOUT.sync = true
 218 
 219 main

Features/StoreID/PredictionAssitenedHelper (last edited 2017-05-23 23:57:22 by Eliezer Croitoru)