This is a helper that receives only the requests domains when the flow into squid and reports them into a centralized logging server. This helper is a part of a suite that analyze requests domains and schedules human inspection of unknown and categorized sites.

   1 package main
   2 
   3 /*
   4 license note
   5 Copyright (c) 2016, Eliezer Croitoru
   6 All rights reserved.
   7 
   8 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
   9 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  10 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
  11 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
  12 
  13 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  14 */
  15 
  16 // Resources:
  17 // - http://www.jokecamp.com/blog/examples-of-creating-base64-hashes-using-hmac-sha256-in-different-languages/
  18 // - https://gobyexample.com/base64-encoding
  19 
  20 // Squid settings
  21 /*
  22 external_acl_type log_doms ipv4 concurrency=1000 ttl=15 %DST %SRC %METHOD /opt/squid-dom-2api -api-url=http://domslog.ngtech.co.il/
  23 acl log_doms_acl external log_doms
  24 http_access deny !log_doms_acl
  25 */
  26 
  27 import (
  28         "bufio"
  29         "crypto/tls"
  30         "crypto/x509"
  31         //      "encoding/base64"
  32         "flag"
  33         "fmt"
  34         "io/ioutil"
  35 
  36         "golang.org/x/net/http2"
  37         //"net"
  38         // Net libs can be used to parse IP addresses
  39         "net/http"
  40         "net/url"
  41         "os"
  42         "strings"
  43 )
  44 
  45 var debug *bool
  46 var domLogApiUrl *string
  47 var user *string
  48 var pass *string
  49 var tlsConfig *tls.Config
  50 var tlsCert *string
  51 var pemCert []byte
  52 var dontVerifyTls *bool
  53 var useOsTls *bool
  54 var http_version *string
  55 var idkey *string
  56 
  57 var err error
  58 
  59 const answer = "OK"
  60 
  61 var client *http.Client
  62 
  63 func process_request(line string) {
  64         lparts := strings.Split(strings.TrimRight(line, "\n"), " ")
  65         fmt.Println(lparts[0] + " " + answer)
  66 
  67         if len(lparts[0]) > 0 {
  68                 if *debug {
  69                         fmt.Fprintln(os.Stderr, "ERRlog: Proccessing request => \""+strings.TrimRight(line, "\n")+"\"")
  70                 }
  71         }
  72 
  73         testurl, _ := url.Parse(*domLogApiUrl)
  74         testurlVals := url.Values{}
  75         testurlVals.Set("domain", lparts[1])
  76         //testurlVals.Set("other", port)
  77         //testurlVals.Set("otherother", srcip)
  78         testurl.RawQuery = testurlVals.Encode()
  79 
  80         request, err := http.NewRequest("GET", testurl.String(), nil)
  81         request.Header.Set("User-Agent", "Golang_Domslog_Bot/1.0_key="+*idkey)
  82         request.Close = true
  83         request.SetBasicAuth(*user, *pass)
  84 
  85         resp, err := client.Do(request)
  86         if err != nil {
  87                 fmt.Fprintln(os.Stderr, "ERRlog: reporting a http connection error1 => \""+err.Error()+"\"")
  88                 fmt.Println(lparts[0] + " " + answer)
  89                 return
  90         }
  91 
  92         defer resp.Body.Close()
  93 
  94         body, err := ioutil.ReadAll(resp.Body)
  95         if err != nil {
  96                 fmt.Fprintln(os.Stderr, "ERRlog: reporting a http connection error2 => \""+err.Error()+"\"")
  97                 fmt.Println(lparts[0] + " " + answer)
  98                 return
  99         }
 100 
 101         if body != nil {
 102                 // Verify that there is no authorization and authentication error
 103         }
 104 }
 105 
 106 func init() {
 107 
 108         fmt.Fprintln(os.Stderr, "ERRlog: Starting Fake helper")
 109 
 110         debug = flag.Bool("d", false, "Debug mode can be \"yes\" or something else for no")
 111         domLogApiUrl = flag.String("api-url", "http://ngtech.co.il/fake-dom-log-api/", "The url of the api")
 112         user = flag.String("api-user", "admin", "Basic auth username for server authentication")
 113         pass = flag.String("api-pass", "admin", "Basic auth password for server authentication")
 114         http_version = flag.String("api-httpv", "1", "http client version: 1\\2")
 115         tlsCert = flag.String("tlscert", "cert.pem", "tls certificate")
 116         dontVerifyTls = flag.Bool("skiptls", false, "Verify tls certificate, use \"1\" to enable")
 117         useOsTls = flag.Bool("ostls", false, "Use OS tls certificates, use \"1\" to enable")
 118         idkey = flag.String("sysuuid", "1", "System UUID key")
 119 
 120         flag.Parse()
 121         flagsMap := make(map[string]interface{})
 122         flagsMap["debug"] = *debug
 123         flagsMap["api-url"] = *domLogApiUrl
 124         flagsMap["api_user"] = *user
 125         flagsMap["api_pass"] = *pass
 126         flagsMap["api-httpv"] = *http_version
 127         flagsMap["tlscert"] = *tlsCert
 128         flagsMap["skiptls"] = *dontVerifyTls
 129         flagsMap["ostls"] = *useOsTls
 130         flagsMap["sysuuid"] = *idkey
 131 
 132         if *debug {
 133                 fmt.Fprintln(os.Stderr, "ERRlog: Config Variables:")
 134                 for k, v := range flagsMap {
 135                         fmt.Fprintf(os.Stderr, "ERRlog:\t%v =>  %v\n", k, v)
 136                 }
 137         }
 138 }
 139 
 140 func main() {
 141         if *http_version == "2" && strings.HasPrefix(*domLogApiUrl, "http://") {
 142                 fmt.Fprintf(os.Stderr, "ERRlog: ### The http2 library doesn't support \"https://\" scheme, you are using => %v\n", *domLogApiUrl)
 143                 return
 144         }
 145 
 146         switch {
 147         case *http_version == "2" && *useOsTls:
 148 
 149         case (*http_version == "2") && !*dontVerifyTls:
 150                 tlsConfig = &tls.Config{RootCAs: x509.NewCertPool()}
 151 
 152                 var err error
 153                 pemCert, err = ioutil.ReadFile(*tlsCert)
 154                 if err != nil {
 155                         fmt.Println(err)
 156                 }
 157                 ok := tlsConfig.RootCAs.AppendCertsFromPEM(pemCert)
 158                 if !ok {
 159                         panic("Couldn't load PEM data")
 160                 }
 161         case (*http_version == "2" && (*dontVerifyTls)):
 162                 tlsConfig = &tls.Config{InsecureSkipVerify: true}
 163         default:
 164 
 165         }
 166 
 167         switch *http_version {
 168         case "2":
 169                 client = &http.Client{
 170                         Transport: &http2.Transport{TLSClientConfig: tlsConfig},
 171                 }
 172         default:
 173                 client = &http.Client{}
 174         }
 175 
 176         reader := bufio.NewReader(os.Stdin)
 177 
 178         for {
 179                 line, err := reader.ReadString('\n')
 180 
 181                 if err != nil {
 182                         // You may check here if err == io.EOF
 183                         break
 184                 }
 185                 if strings.HasPrefix(line, "q") || strings.HasPrefix(line, "Q") {
 186                         fmt.Fprintln(os.Stderr, "ERRlog: Exiting cleanly")
 187                         break
 188                 }
 189                 // Will return an answer as soon as possible
 190                 go process_request(line)
 191 
 192         }
 193 }

EliezerCroitoru/Helpers/DomainsLogger (last edited 2016-07-26 23:10:33 by Eliezer Croitoru)