From 48fa0d6642360e384839fd0485a21e6781d7a047 Mon Sep 17 00:00:00 2001 From: Philippe-Adrien Nousse Date: Thu, 5 Jul 2018 12:50:56 +0200 Subject: [PATCH] Nextcloud API Client, Uers/Response types, User list/search/get/create/delete --- .gitignore | 69 + README.md | 85 + client/apps.go | 1 + client/auth.go | 37 + client/client.go | 31 + client/groups.go | 1 + client/routes.go | 19 + client/types/capabilities.go | 94 + client/types/responses.go | 79 + client/types/user.go | 21 + client/users.go | 103 + gonextcloud.iml | 9 + .../github.com/google/go-querystring/LICENSE | 27 + .../google/go-querystring/query/encode.go | 320 + vendor/github.com/levigross/grequests/LICENSE | 202 + .../github.com/levigross/grequests/README.md | 776 ++ vendor/github.com/levigross/grequests/base.go | 67 + .../levigross/grequests/file_upload.go | 68 + .../github.com/levigross/grequests/request.go | 585 + .../levigross/grequests/response.go | 215 + .../github.com/levigross/grequests/session.go | 136 + .../github.com/levigross/grequests/utils.go | 102 + vendor/golang.org/x/net/LICENSE | 27 + vendor/golang.org/x/net/PATENTS | 22 + vendor/golang.org/x/net/publicsuffix/gen.go | 713 ++ vendor/golang.org/x/net/publicsuffix/list.go | 135 + vendor/golang.org/x/net/publicsuffix/table.go | 9745 +++++++++++++++++ vendor/vendor.json | 25 + 28 files changed, 13714 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 client/apps.go create mode 100644 client/auth.go create mode 100644 client/client.go create mode 100644 client/groups.go create mode 100644 client/routes.go create mode 100644 client/types/capabilities.go create mode 100644 client/types/responses.go create mode 100644 client/types/user.go create mode 100644 client/users.go create mode 100644 gonextcloud.iml create mode 100644 vendor/github.com/google/go-querystring/LICENSE create mode 100644 vendor/github.com/google/go-querystring/query/encode.go create mode 100644 vendor/github.com/levigross/grequests/LICENSE create mode 100644 vendor/github.com/levigross/grequests/README.md create mode 100644 vendor/github.com/levigross/grequests/base.go create mode 100644 vendor/github.com/levigross/grequests/file_upload.go create mode 100644 vendor/github.com/levigross/grequests/request.go create mode 100644 vendor/github.com/levigross/grequests/response.go create mode 100644 vendor/github.com/levigross/grequests/session.go create mode 100644 vendor/github.com/levigross/grequests/utils.go create mode 100644 vendor/golang.org/x/net/LICENSE create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/publicsuffix/gen.go create mode 100644 vendor/golang.org/x/net/publicsuffix/list.go create mode 100644 vendor/golang.org/x/net/publicsuffix/table.go create mode 100644 vendor/vendor.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c6a378 --- /dev/null +++ b/.gitignore @@ -0,0 +1,69 @@ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +.idea +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/dictionaries +.idea/**/shelf + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +main.go \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4a4215 --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +# [WIP] Nextcloud Go API Client + +A simple Client for Nextcloud's API in Go. + +## TODO +- [Auth](#authentication) + - ~~login~~ + - ~~logout~~ +- [Users](#users) + - ~~search~~ + - ~~list~~ + - ~~get infos~~ + - ~~create~~ + - update + - ~~delete~~ + - enable + - disable + - get groups + - add to group + - remove from group + - get subadmin group + - promote subadmin + - demote subadmin + - send welcome mail +- [Groups](#groups) + - create + - delete + - get members + - get subadmins +- [Apps](#apps) + - list + - get infos + - enable + - disable + +# Getting started +## Authentication +```go +package main + +import ( + "fmt" + "github.com/partitio/gonextcloud/client" +) + +func main() { + url := "https://www.mynextcloud.com" + username := "admin" + password := "password" + c, err := client.NewClient(url) + if err != nil { + panic(err) + } + if err := c.Login(username, password); err != nil { + panic(err) + } + defer c.Logout() +} +```` +##Users +List : +```go +func (c *Client) UserList() ([]string, error) +``` +Search +```go +func (c *Client) UserSearch(search string) ([]string, error) +``` +Get +```go +func (c *Client) User(name string) (*types.User, error) +``` +Create +```go +func (c *Client) UserCreate(username string, password string) error +``` +Delete +```go +func (c *Client) UserDelete(name string) error +``` +## Groups +TODO + +## Apps +TODO \ No newline at end of file diff --git a/client/apps.go b/client/apps.go new file mode 100644 index 0000000..da13c8e --- /dev/null +++ b/client/apps.go @@ -0,0 +1 @@ +package client diff --git a/client/auth.go b/client/auth.go new file mode 100644 index 0000000..9bc782f --- /dev/null +++ b/client/auth.go @@ -0,0 +1,37 @@ +package client + +import ( + "fmt" + req "github.com/levigross/grequests" + "github.com/partitio/gonextcloud/client/types" +) + +var unauthorized = fmt.Errorf("login first") + +func (c *Client) Login(username string, password string) error { + c.username = username + c.password = password + options := req.RequestOptions{ + Headers: c.headers, + Auth: []string{c.username, c.password}, + } + c.session = req.NewSession(&options) + u := c.baseURL.ResolveReference(routes.capabilities) + r, err := c.session.Get(u.String(), nil) + if err != nil { + return err + } + var cs types.CapabilitiesResponse + r.JSON(&cs) + c.capabilities = &cs.Ocs.Data.Capabilities + return nil +} + +func (c *Client) Logout() error { + c.session.CloseIdleConnections() + return nil +} + +func (c *Client) loggedIn() bool { + return c.capabilities != nil +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..6224a86 --- /dev/null +++ b/client/client.go @@ -0,0 +1,31 @@ +package client + +import ( + req "github.com/levigross/grequests" + "github.com/partitio/gonextcloud/client/types" + "net/url" +) + +type Client struct { + baseURL *url.URL + username string + password string + session *req.Session + headers map[string]string + capabilities *types.Capabilities +} + +func NewClient(hostname string) (*Client, error) { + baseURL, err := url.Parse(hostname) + if err != nil { + return nil, err + } + c := Client{ + baseURL: baseURL, + headers: map[string]string{ + "OCS-APIREQUEST": "true", + "Accept": "application/json", + }, + } + return &c, nil +} diff --git a/client/groups.go b/client/groups.go new file mode 100644 index 0000000..da13c8e --- /dev/null +++ b/client/groups.go @@ -0,0 +1 @@ +package client diff --git a/client/routes.go b/client/routes.go new file mode 100644 index 0000000..24ba993 --- /dev/null +++ b/client/routes.go @@ -0,0 +1,19 @@ +package client + +import "net/url" + +type Routes struct { + capabilities *url.URL + users *url.URL + groups *url.URL +} + +var ( + apiPath = &url.URL{Path: "/ocs/v1.php/cloud"} + routes = Routes{ + capabilities: &url.URL{Path: apiPath.Path + "/capabilities"}, + users: &url.URL{Path: apiPath.Path + "/users"}, + groups: &url.URL{Path: apiPath.Path + "/groups"}, + } + badRequest = 998 +) diff --git a/client/types/capabilities.go b/client/types/capabilities.go new file mode 100644 index 0000000..fba7d7b --- /dev/null +++ b/client/types/capabilities.go @@ -0,0 +1,94 @@ +package types + +type Capabilities struct { + Core struct { + Pollinterval int `json:"pollinterval"` + WebdavRoot string `json:"webdav-root"` + } `json:"core"` + Bruteforce struct { + Delay int `json:"delay"` + } `json:"bruteforce"` + Activity struct { + Apiv2 []string `json:"apiv2"` + } `json:"activity"` + Dav struct { + Chunking string `json:"chunking"` + } `json:"dav"` + FilesSharing struct { + APIEnabled bool `json:"api_enabled"` + Public struct { + Enabled bool `json:"enabled"` + Password struct { + Enforced bool `json:"enforced"` + } `json:"password"` + ExpireDate struct { + Enabled bool `json:"enabled"` + } `json:"expire_date"` + SendMail bool `json:"send_mail"` + Upload bool `json:"upload"` + UploadFilesDrop bool `json:"upload_files_drop"` + } `json:"public"` + Resharing bool `json:"resharing"` + User struct { + SendMail bool `json:"send_mail"` + ExpireDate struct { + Enabled bool `json:"enabled"` + } `json:"expire_date"` + } `json:"user"` + GroupSharing bool `json:"group_sharing"` + Group struct { + Enabled bool `json:"enabled"` + ExpireDate struct { + Enabled bool `json:"enabled"` + } `json:"expire_date"` + } `json:"group"` + Federation struct { + Outgoing bool `json:"outgoing"` + Incoming bool `json:"incoming"` + ExpireDate struct { + Enabled bool `json:"enabled"` + } `json:"expire_date"` + } `json:"federation"` + Sharebymail struct { + Enabled bool `json:"enabled"` + UploadFilesDrop struct { + Enabled bool `json:"enabled"` + } `json:"upload_files_drop"` + Password struct { + Enabled bool `json:"enabled"` + } `json:"password"` + ExpireDate struct { + Enabled bool `json:"enabled"` + } `json:"expire_date"` + } `json:"sharebymail"` + } `json:"files_sharing"` + Notifications struct { + OcsEndpoints []string `json:"ocs-endpoints"` + Push []string `json:"push"` + } `json:"notifications"` + PasswordPolicy struct { + MinLength int `json:"minLength"` + EnforceNonCommonPassword bool `json:"enforceNonCommonPassword"` + EnforceNumericCharacters bool `json:"enforceNumericCharacters"` + EnforceSpecialCharacters bool `json:"enforceSpecialCharacters"` + EnforceUpperLowerCase bool `json:"enforceUpperLowerCase"` + } `json:"password_policy"` + Theming struct { + Name string `json:"name"` + URL string `json:"url"` + Slogan string `json:"slogan"` + Color string `json:"color"` + ColorText string `json:"color-text"` + ColorElement string `json:"color-element"` + Logo string `json:"logo"` + Background string `json:"background"` + BackgroundPlain bool `json:"background-plain"` + BackgroundDefault bool `json:"background-default"` + } `json:"theming"` + Files struct { + Bigfilechunking bool `json:"bigfilechunking"` + BlacklistedFiles []string `json:"blacklisted_files"` + Undelete bool `json:"undelete"` + Versioning bool `json:"versioning"` + } `json:"files"` +} diff --git a/client/types/responses.go b/client/types/responses.go new file mode 100644 index 0000000..1bc9b7a --- /dev/null +++ b/client/types/responses.go @@ -0,0 +1,79 @@ +package types + +type ErrorResponse struct { + Ocs struct { + Meta struct { + Status string `json:"status"` + Statuscode int `json:"statuscode"` + Message string `json:"message"` + Totalitems string `json:"totalitems"` + Itemsperpage string `json:"itemsperpage"` + } `json:"meta"` + Data []interface{} `json:"data"` + } `json:"ocs"` +} + +type UserListResponse struct { + Ocs struct { + Meta struct { + Status string `json:"status"` + Statuscode int `json:"statuscode"` + Message string `json:"message"` + Totalitems string `json:"totalitems"` + Itemsperpage string `json:"itemsperpage"` + } `json:"meta"` + Data struct { + Users []string `json:"users"` + } `json:"data"` + } `json:"ocs"` +} + +type UserResponse struct { + Ocs struct { + Meta struct { + Status string `json:"status"` + Statuscode int `json:"statuscode"` + Message string `json:"message"` + Totalitems string `json:"totalitems"` + Itemsperpage string `json:"itemsperpage"` + } `json:"meta"` + Data User `json:"data"` + } `json:"ocs"` +} + +type GroupListResponse struct { + Ocs struct { + Meta struct { + Status string `json:"status"` + Statuscode int `json:"statuscode"` + Message string `json:"message"` + Totalitems string `json:"totalitems"` + Itemsperpage string `json:"itemsperpage"` + } `json:"meta"` + Data struct { + Groups []string `json:"groups"` + } `json:"data"` + } `json:"ocs"` +} + +type CapabilitiesResponse struct { + Ocs struct { + Meta struct { + Status string `json:"status"` + Statuscode int `json:"statuscode"` + Message string `json:"message"` + Totalitems string `json:"totalitems"` + Itemsperpage string `json:"itemsperpage"` + } `json:"meta"` + Data struct { + Version struct { + Major int `json:"major"` + Minor int `json:"minor"` + Micro int `json:"micro"` + String string `json:"string"` + Edition string `json:"edition"` + } `json:"version"` + Capabilities Capabilities `json:"capabilities"` + } `json:"data"` + } `json:"ocs"` +} diff --git a/client/types/user.go b/client/types/user.go new file mode 100644 index 0000000..c749dc2 --- /dev/null +++ b/client/types/user.go @@ -0,0 +1,21 @@ +package types + +type User struct { + Enabled string `json:"enabled"` + ID string `json:"id"` + Quota struct { + Free int64 `json:"free"` + Used int `json:"used"` + Total int64 `json:"total"` + Relative float64 `json:"relative"` + Quota int `json:"quota"` + } `json:"quota"` + Email string `json:"email"` + Displayname string `json:"displayname"` + Phone string `json:"phone"` + Address string `json:"address"` + Website string `json:"website"` + Twitter string `json:"twitter"` + Groups []string `json:"groups"` + Language string `json:"language"` +} diff --git a/client/users.go b/client/users.go new file mode 100644 index 0000000..f5b91d8 --- /dev/null +++ b/client/users.go @@ -0,0 +1,103 @@ +package client + +import ( + "fmt" + req "github.com/levigross/grequests" + "github.com/partitio/gonextcloud/client/types" + "path" +) + +func (c *Client) UserList() ([]string, error) { + if !c.loggedIn() { + return nil, unauthorized + } + u := c.baseURL.ResolveReference(routes.users) + res, err := c.session.Get(u.String(), nil) + if err != nil { + return nil, err + } + var ul types.UserListResponse + res.JSON(&ul) + return ul.Ocs.Data.Users, nil +} + +func (c *Client) User(name string) (*types.User, error) { + if !c.loggedIn() { + return nil, unauthorized + } + u := c.baseURL.ResolveReference(routes.users) + u.Path = path.Join(u.Path, name) + res, err := c.session.Get(u.String(), nil) + if err != nil { + return nil, err + } + var ur types.UserResponse + res.JSON(&ur) + if ur.Ocs.Meta.Statuscode != 100 { + return nil, fmt.Errorf(ur.Ocs.Meta.Message) + } + return &ur.Ocs.Data, nil +} + +func (c *Client) UserSearch(search string) ([]string, error) { + if !c.loggedIn() { + return nil, unauthorized + } + u := c.baseURL.ResolveReference(routes.users) + ro := &req.RequestOptions{ + Params: map[string]string{"search": search}, + } + res, err := c.session.Get(u.String(), ro) + if err != nil { + return nil, err + } + var r types.UserListResponse + res.JSON(&r) + if r.Ocs.Meta.Statuscode != 100 { + return nil, fmt.Errorf(r.Ocs.Meta.Message) + } + return r.Ocs.Data.Users, nil +} + +func (c *Client) UserCreate(username string, password string) error { + if !c.loggedIn() { + return unauthorized + } + u := c.baseURL.ResolveReference(routes.users) + ro := &req.RequestOptions{ + Data: map[string]string{ + "userid": username, + "password": password, + }, + } + res, err := c.session.Post(u.String(), ro) + if err != nil { + return err + } + fmt.Println(res.String()) + var r types.UserResponse + res.JSON(&r) + if r.Ocs.Meta.Statuscode != 100 { + return fmt.Errorf(r.Ocs.Meta.Message) + } + return nil +} + +func (c *Client) UserDelete(name string) error { + if !c.loggedIn() { + return unauthorized + } + u := c.baseURL.ResolveReference(routes.users) + u.Path = path.Join(u.Path, name) + res, err := c.session.Delete(u.String(), nil) + if err != nil { + return err + } + var ur types.UserResponse + fmt.Println(res.String()) + res.JSON(&ur) + if ur.Ocs.Meta.Statuscode != 100 { + return fmt.Errorf(ur.Ocs.Meta.Message) + } + return nil +} diff --git a/gonextcloud.iml b/gonextcloud.iml new file mode 100644 index 0000000..eacc75a --- /dev/null +++ b/gonextcloud.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/vendor/github.com/google/go-querystring/LICENSE b/vendor/github.com/google/go-querystring/LICENSE new file mode 100644 index 0000000..ae121a1 --- /dev/null +++ b/vendor/github.com/google/go-querystring/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Google. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/go-querystring/query/encode.go b/vendor/github.com/google/go-querystring/query/encode.go new file mode 100644 index 0000000..37080b1 --- /dev/null +++ b/vendor/github.com/google/go-querystring/query/encode.go @@ -0,0 +1,320 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package query implements encoding of structs into URL query parameters. +// +// As a simple example: +// +// type Options struct { +// Query string `url:"q"` +// ShowAll bool `url:"all"` +// Page int `url:"page"` +// } +// +// opt := Options{ "foo", true, 2 } +// v, _ := query.Values(opt) +// fmt.Print(v.Encode()) // will output: "q=foo&all=true&page=2" +// +// The exact mapping between Go values and url.Values is described in the +// documentation for the Values() function. +package query + +import ( + "bytes" + "fmt" + "net/url" + "reflect" + "strconv" + "strings" + "time" +) + +var timeType = reflect.TypeOf(time.Time{}) + +var encoderType = reflect.TypeOf(new(Encoder)).Elem() + +// Encoder is an interface implemented by any type that wishes to encode +// itself into URL values in a non-standard way. +type Encoder interface { + EncodeValues(key string, v *url.Values) error +} + +// Values returns the url.Values encoding of v. +// +// Values expects to be passed a struct, and traverses it recursively using the +// following encoding rules. +// +// Each exported struct field is encoded as a URL parameter unless +// +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option +// +// The empty values are false, 0, any nil pointer or interface value, any array +// slice, map, or string of length zero, and any time.Time that returns true +// for IsZero(). +// +// The URL parameter name defaults to the struct field name but can be +// specified in the struct field's tag value. The "url" key in the struct +// field's tag value is the key name, followed by an optional comma and +// options. For example: +// +// // Field is ignored by this package. +// Field int `url:"-"` +// +// // Field appears as URL parameter "myName". +// Field int `url:"myName"` +// +// // Field appears as URL parameter "myName" and the field is omitted if +// // its value is empty +// Field int `url:"myName,omitempty"` +// +// // Field appears as URL parameter "Field" (the default), but the field +// // is skipped if empty. Note the leading comma. +// Field int `url:",omitempty"` +// +// For encoding individual field values, the following type-dependent rules +// apply: +// +// Boolean values default to encoding as the strings "true" or "false". +// Including the "int" option signals that the field should be encoded as the +// strings "1" or "0". +// +// time.Time values default to encoding as RFC3339 timestamps. Including the +// "unix" option signals that the field should be encoded as a Unix time (see +// time.Unix()) +// +// Slice and Array values default to encoding as multiple URL values of the +// same name. Including the "comma" option signals that the field should be +// encoded as a single comma-delimited value. Including the "space" option +// similarly encodes the value as a single space-delimited string. Including +// the "semicolon" option will encode the value as a semicolon-delimited string. +// Including the "brackets" option signals that the multiple URL values should +// have "[]" appended to the value name. "numbered" will append a number to +// the end of each incidence of the value name, example: +// name0=value0&name1=value1, etc. +// +// Anonymous struct fields are usually encoded as if their inner exported +// fields were fields in the outer struct, subject to the standard Go +// visibility rules. An anonymous struct field with a name given in its URL +// tag is treated as having that name, rather than being anonymous. +// +// Non-nil pointer values are encoded as the value pointed to. +// +// Nested structs are encoded including parent fields in value names for +// scoping. e.g: +// +// "user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO" +// +// All other values are encoded using their default string representation. +// +// Multiple fields that encode to the same URL parameter name will be included +// as multiple URL values of the same name. +func Values(v interface{}) (url.Values, error) { + values := make(url.Values) + val := reflect.ValueOf(v) + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return values, nil + } + val = val.Elem() + } + + if v == nil { + return values, nil + } + + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind()) + } + + err := reflectValue(values, val, "") + return values, err +} + +// reflectValue populates the values parameter from the struct fields in val. +// Embedded structs are followed recursively (using the rules defined in the +// Values function documentation) breadth-first. +func reflectValue(values url.Values, val reflect.Value, scope string) error { + var embedded []reflect.Value + + typ := val.Type() + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + + sv := val.Field(i) + tag := sf.Tag.Get("url") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if name == "" { + if sf.Anonymous && sv.Kind() == reflect.Struct { + // save embedded struct for later processing + embedded = append(embedded, sv) + continue + } + + name = sf.Name + } + + if scope != "" { + name = scope + "[" + name + "]" + } + + if opts.Contains("omitempty") && isEmptyValue(sv) { + continue + } + + if sv.Type().Implements(encoderType) { + if !reflect.Indirect(sv).IsValid() { + sv = reflect.New(sv.Type().Elem()) + } + + m := sv.Interface().(Encoder) + if err := m.EncodeValues(name, &values); err != nil { + return err + } + continue + } + + if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array { + var del byte + if opts.Contains("comma") { + del = ',' + } else if opts.Contains("space") { + del = ' ' + } else if opts.Contains("semicolon") { + del = ';' + } else if opts.Contains("brackets") { + name = name + "[]" + } + + if del != 0 { + s := new(bytes.Buffer) + first := true + for i := 0; i < sv.Len(); i++ { + if first { + first = false + } else { + s.WriteByte(del) + } + s.WriteString(valueString(sv.Index(i), opts)) + } + values.Add(name, s.String()) + } else { + for i := 0; i < sv.Len(); i++ { + k := name + if opts.Contains("numbered") { + k = fmt.Sprintf("%s%d", name, i) + } + values.Add(k, valueString(sv.Index(i), opts)) + } + } + continue + } + + for sv.Kind() == reflect.Ptr { + if sv.IsNil() { + break + } + sv = sv.Elem() + } + + if sv.Type() == timeType { + values.Add(name, valueString(sv, opts)) + continue + } + + if sv.Kind() == reflect.Struct { + reflectValue(values, sv, name) + continue + } + + values.Add(name, valueString(sv, opts)) + } + + for _, f := range embedded { + if err := reflectValue(values, f, scope); err != nil { + return err + } + } + + return nil +} + +// valueString returns the string representation of a value. +func valueString(v reflect.Value, opts tagOptions) string { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return "" + } + v = v.Elem() + } + + if v.Kind() == reflect.Bool && opts.Contains("int") { + if v.Bool() { + return "1" + } + return "0" + } + + if v.Type() == timeType { + t := v.Interface().(time.Time) + if opts.Contains("unix") { + return strconv.FormatInt(t.Unix(), 10) + } + return t.Format(time.RFC3339) + } + + return fmt.Sprint(v.Interface()) +} + +// isEmptyValue checks if a value should be considered empty for the purposes +// of omitting fields with the "omitempty" option. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + + if v.Type() == timeType { + return v.Interface().(time.Time).IsZero() + } + + return false +} + +// tagOptions is the string following a comma in a struct field's "url" tag, or +// the empty string. It does not include the leading comma. +type tagOptions []string + +// parseTag splits a struct field's url tag into its name and comma-separated +// options. +func parseTag(tag string) (string, tagOptions) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// Contains checks whether the tagOptions contains the specified option. +func (o tagOptions) Contains(option string) bool { + for _, s := range o { + if s == option { + return true + } + } + return false +} diff --git a/vendor/github.com/levigross/grequests/LICENSE b/vendor/github.com/levigross/grequests/LICENSE new file mode 100644 index 0000000..1a0a94d --- /dev/null +++ b/vendor/github.com/levigross/grequests/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 – Levi Gross + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/levigross/grequests/README.md b/vendor/github.com/levigross/grequests/README.md new file mode 100644 index 0000000..ea5d971 --- /dev/null +++ b/vendor/github.com/levigross/grequests/README.md @@ -0,0 +1,776 @@ +# GRequests +A Go "clone" of the great and famous Requests library + +[![Build Status](https://travis-ci.org/levigross/grequests.svg?branch=master)](https://travis-ci.org/levigross/grequests) [![GoDoc](https://godoc.org/github.com/levigross/grequests?status.svg)](https://godoc.org/github.com/levigross/grequests) +[![Coverage Status](https://coveralls.io/repos/levigross/grequests/badge.svg)](https://coveralls.io/r/levigross/grequests) +[![Join the chat at https://gitter.im/levigross/grequests](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/levigross/grequests?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +License +====== + +GRequests is licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for the full license text + +Features +======== + +- Responses can be serialized into JSON and XML +- Easy file uploads +- Easy file downloads +- Support for the following HTTP verbs `GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONS` + +Install +======= +`go get -u github.com/levigross/grequests` + +Usage +====== +`import "github.com/levigross/grequests"` + +Basic Examples +========= +Basic GET request: + +```go +resp, err := grequests.Get("http://httpbin.org/get", nil) +// You can modify the request by passing an optional RequestOptions struct + +if err != nil { + log.Fatalln("Unable to make request: ", err) +} + +fmt.Println(resp.String()) +// { +// "args": {}, +// "headers": { +// "Accept": "*/*", +// "Host": "httpbin.org", +``` + +If an error occurs all of the other properties and methods of a `Response` will be `nil` + +Quirks +======= +## Request Quirks + +When passing parameters to be added to a URL, if the URL has existing parameters that *_contradict_* with what has been passed within `Params` – `Params` will be the "source of authority" and overwrite the contradicting URL parameter. + +Lets see how it works... + +```go +ro := &RequestOptions{ + Params: map[string]string{"Hello": "Goodbye"}, +} +Get("http://httpbin.org/get?Hello=World", ro) +// The URL is now http://httpbin.org/get?Hello=Goodbye +``` + +## Response Quirks + +Order matters! This is because `grequests.Response` is implemented as an `io.ReadCloser` which proxies the *http.Response.Body* `io.ReadCloser` interface. It also includes an internal buffer for use in `Response.String()` and `Response.Bytes()`. + +Here are a list of methods that consume the *http.Response.Body* `io.ReadCloser` interface. + +- Response.JSON +- Response.XML +- Response.DownloadToFile +- Response.Close +- Response.Read + +The following methods make use of an internal byte buffer + +- Response.String +- Response.Bytes + +In the code below, once the file is downloaded – the `Response` struct no longer has access to the request bytes + +```go +response := Get("http://some-wonderful-file.txt", nil) + +if err := response.DownloadToFile("randomFile"); err != nil { + log.Println("Unable to download file: ", err) +} + +// At this point the .String and .Bytes method will return empty responses + +response.Bytes() == nil // true +response.String() == "" // true + +``` + +But if we were to call `response.Bytes()` or `response.String()` first, every operation will succeed until the internal buffer is cleared: + +```go +response := Get("http://some-wonderful-file.txt", nil) + +// This call to .Bytes caches the request bytes in an internal byte buffer – which can be used again and again until it is cleared +response.Bytes() == `file-bytes` +response.String() == "file-string" + +// This will work because it will use the internal byte buffer +if err := resp.DownloadToFile("randomFile"); err != nil { + log.Println("Unable to download file: ", err) +} + +// Now if we clear the internal buffer.... +response.ClearInternalBuffer() + +// At this point the .String and .Bytes method will return empty responses + +response.Bytes() == nil // true +response.String() == "" // true +``` + + + + +# grequests +`import "github.com/levigross/grequests"` + +* [Overview](#pkg-overview) +* [Index](#pkg-index) +* [Examples](#pkg-examples) + +## Overview +Package grequests implements a friendly API over Go's existing net/http library + + + + +## Index +* [Variables](#pkg-variables) +* [func BuildHTTPClient(ro RequestOptions) *http.Client](#BuildHTTPClient) +* [func EnsureTransporterFinalized(httpTransport *http.Transport)](#EnsureTransporterFinalized) +* [func FileUploadFromDisk(fileName string) ([]FileUpload, error)](#FileUploadFromDisk) +* [func FileUploadFromGlob(fileSystemGlob string) ([]FileUpload, error)](#FileUploadFromGlob) +* [type FileUpload](#FileUpload) +* [type RequestOptions](#RequestOptions) +* [type Response](#Response) + * [func Delete(url string, ro *RequestOptions) (*Response, error)](#Delete) + * [func Get(url string, ro *RequestOptions) (*Response, error)](#Get) + * [func Head(url string, ro *RequestOptions) (*Response, error)](#Head) + * [func Options(url string, ro *RequestOptions) (*Response, error)](#Options) + * [func Patch(url string, ro *RequestOptions) (*Response, error)](#Patch) + * [func Post(url string, ro *RequestOptions) (*Response, error)](#Post) + * [func Put(url string, ro *RequestOptions) (*Response, error)](#Put) + * [func (r *Response) Bytes() []byte](#Response.Bytes) + * [func (r *Response) ClearInternalBuffer()](#Response.ClearInternalBuffer) + * [func (r *Response) Close() error](#Response.Close) + * [func (r *Response) DownloadToFile(fileName string) error](#Response.DownloadToFile) + * [func (r *Response) JSON(userStruct interface{}) error](#Response.JSON) + * [func (r *Response) Read(p []byte) (n int, err error)](#Response.Read) + * [func (r *Response) String() string](#Response.String) + * [func (r *Response) XML(userStruct interface{}, charsetReader XMLCharDecoder) error](#Response.XML) +* [type Session](#Session) + * [func NewSession(ro *RequestOptions) *Session](#NewSession) + * [func (s *Session) CloseIdleConnections()](#Session.CloseIdleConnections) + * [func (s *Session) Delete(url string, ro *RequestOptions) (*Response, error)](#Session.Delete) + * [func (s *Session) Get(url string, ro *RequestOptions) (*Response, error)](#Session.Get) + * [func (s *Session) Head(url string, ro *RequestOptions) (*Response, error)](#Session.Head) + * [func (s *Session) Options(url string, ro *RequestOptions) (*Response, error)](#Session.Options) + * [func (s *Session) Patch(url string, ro *RequestOptions) (*Response, error)](#Session.Patch) + * [func (s *Session) Post(url string, ro *RequestOptions) (*Response, error)](#Session.Post) + * [func (s *Session) Put(url string, ro *RequestOptions) (*Response, error)](#Session.Put) +* [type XMLCharDecoder](#XMLCharDecoder) + +#### Examples +* [Package (AcceptInvalidTLSCert)](#example__acceptInvalidTLSCert) +* [Package (BasicAuth)](#example__basicAuth) +* [Package (BasicGet)](#example__basicGet) +* [Package (BasicGetCustomHTTPClient)](#example__basicGetCustomHTTPClient) +* [Package (Cookies)](#example__cookies) +* [Package (CustomHTTPHeader)](#example__customHTTPHeader) +* [Package (CustomUserAgent)](#example__customUserAgent) +* [Package (DownloadFile)](#example__downloadFile) +* [Package (PostFileUpload)](#example__postFileUpload) +* [Package (PostForm)](#example__postForm) +* [Package (PostJSONAJAX)](#example__postJSONAJAX) +* [Package (PostXML)](#example__postXML) +* [Package (Proxy)](#example__proxy) +* [Package (Session)](#example__session) +* [Package (UrlQueryParams)](#example__urlQueryParams) + +#### Package files +[base.go](/src/github.com/levigross/grequests/base.go) [file_upload.go](/src/github.com/levigross/grequests/file_upload.go) [request.go](/src/github.com/levigross/grequests/request.go) [response.go](/src/github.com/levigross/grequests/response.go) [session.go](/src/github.com/levigross/grequests/session.go) [utils.go](/src/github.com/levigross/grequests/utils.go) + + + +## Variables +``` go +var ( + // ErrRedirectLimitExceeded is the error returned when the request responded + // with too many redirects + ErrRedirectLimitExceeded = errors.New("grequests: Request exceeded redirect count") + + // RedirectLimit is a tunable variable that specifies how many times we can + // redirect in response to a redirect. This is the global variable, if you + // wish to set this on a request by request basis, set it within the + // `RequestOptions` structure + RedirectLimit = 30 + + // SensitiveHTTPHeaders is a map of sensitive HTTP headers that a user + // doesn't want passed on a redirect. This is the global variable, if you + // wish to set this on a request by request basis, set it within the + // `RequestOptions` structure + SensitiveHTTPHeaders = map[string]struct{}{ + "Www-Authenticate": {}, + "Authorization": {}, + "Proxy-Authorization": {}, + } +) +``` + + +## func [BuildHTTPClient](/src/target/request.go?s=11478:11530#L405) +``` go +func BuildHTTPClient(ro RequestOptions) *http.Client +``` +BuildHTTPClient is a function that will return a custom HTTP client based on the request options provided +the check is in UseDefaultClient + + + +## func [EnsureTransporterFinalized](/src/target/utils.go?s=2482:2544#L79) +``` go +func EnsureTransporterFinalized(httpTransport *http.Transport) +``` +EnsureTransporterFinalized will ensure that when the HTTP client is GCed +the runtime will close the idle connections (so that they won't leak) +this function was adopted from Hashicorp's go-cleanhttp package + + + +## func [FileUploadFromDisk](/src/target/file_upload.go?s=625:687#L14) +``` go +func FileUploadFromDisk(fileName string) ([]FileUpload, error) +``` +FileUploadFromDisk allows you to create a FileUpload struct slice by just specifying a location on the disk + + + +## func [FileUploadFromGlob](/src/target/file_upload.go?s=1068:1136#L27) +``` go +func FileUploadFromGlob(fileSystemGlob string) ([]FileUpload, error) +``` +FileUploadFromGlob allows you to create a FileUpload struct slice by just specifying a glob location on the disk +this function will gloss over all errors in the files and only upload the files that don't return errors from the glob + + + + +## type [FileUpload](/src/target/file_upload.go?s=162:512#L2) +``` go +type FileUpload struct { + // Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server + FileName string + + // FileContents is happy as long as you pass it a io.ReadCloser (which most file use anyways) + FileContents io.ReadCloser + + // FieldName is form field name + FieldName string +} +``` +FileUpload is a struct that is used to specify the file that a User +wishes to upload. + + + + + + + + + + +## type [RequestOptions](/src/target/request.go?s=357:4258#L18) +``` go +type RequestOptions struct { + + // Data is a map of key values that will eventually convert into the + // query string of a GET request or the body of a POST request. + Data map[string]string + + // Params is a map of query strings that may be used within a GET request + Params map[string]string + + // QueryStruct is a struct that encapsulates a set of URL query params + // this paramter is mutually exclusive with `Params map[string]string` (they cannot be combined) + // for more information please see https://godoc.org/github.com/google/go-querystring/query + QueryStruct interface{} + + // Files is where you can include files to upload. The use of this data + // structure is limited to POST requests + Files []FileUpload + + // JSON can be used when you wish to send JSON within the request body + JSON interface{} + + // XML can be used if you wish to send XML within the request body + XML interface{} + + // Headers if you want to add custom HTTP headers to the request, + // this is your friend + Headers map[string]string + + // InsecureSkipVerify is a flag that specifies if we should validate the + // server's TLS certificate. It should be noted that Go's TLS verify mechanism + // doesn't validate if a certificate has been revoked + InsecureSkipVerify bool + + // DisableCompression will disable gzip compression on requests + DisableCompression bool + + // UserAgent allows you to set an arbitrary custom user agent + UserAgent string + + // Host allows you to set an arbitrary custom host + Host string + + // Auth allows you to specify a user name and password that you wish to + // use when requesting the URL. It will use basic HTTP authentication + // formatting the username and password in base64 the format is: + // []string{username, password} + Auth []string + + // IsAjax is a flag that can be set to make the request appear + // to be generated by browser Javascript + IsAjax bool + + // Cookies is an array of `http.Cookie` that allows you to attach + // cookies to your request + Cookies []*http.Cookie + + // UseCookieJar will create a custom HTTP client that will + // process and store HTTP cookies when they are sent down + UseCookieJar bool + + // Proxies is a map in the following format + // *protocol* => proxy address e.g http => http://127.0.0.1:8080 + Proxies map[string]*url.URL + + // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // wait for a TLS handshake. Zero means no timeout. + TLSHandshakeTimeout time.Duration + + // DialTimeout is the maximum amount of time a dial will wait for + // a connect to complete. + DialTimeout time.Duration + + // KeepAlive specifies the keep-alive period for an active + // network connection. If zero, keep-alive are not enabled. + DialKeepAlive time.Duration + + // RequestTimeout is the maximum amount of time a whole request(include dial / request / redirect) + // will wait. + RequestTimeout time.Duration + + // HTTPClient can be provided if you wish to supply a custom HTTP client + // this is useful if you want to use an OAUTH client with your request. + HTTPClient *http.Client + + // SensitiveHTTPHeaders is a map of sensitive HTTP headers that a user + // doesn't want passed on a redirect. + SensitiveHTTPHeaders map[string]struct{} + + // RedirectLimit is the acceptable amount of redirects that we should expect + // before returning an error be default this is set to 30. You can change this + // globally by modifying the `RedirectLimit` variable. + RedirectLimit int + + // RequestBody allows you to put anything matching an `io.Reader` into the request + // this option will take precedence over any other request option specified + RequestBody io.Reader + + // CookieJar allows you to specify a special cookiejar to use with your request. + // this option will take precedence over the `UseCookieJar` option above. + CookieJar http.CookieJar + + // Context can be used to maintain state between requests https://golang.org/pkg/context/#Context + Context context.Context +} +``` +RequestOptions is the location that of where the data + + + + + + + + + + +## type [Response](/src/target/response.go?s=181:806#L4) +``` go +type Response struct { + + // Ok is a boolean flag that validates that the server returned a 2xx code + Ok bool + + // This is the Go error flag – if something went wrong within the request, this flag will be set. + Error error + + // We want to abstract (at least at the moment) the Go http.Response object away. So we are going to make use of it + // internal but not give the user access + RawResponse *http.Response + + // StatusCode is the HTTP Status Code returned by the HTTP Response. Taken from resp.StatusCode + StatusCode int + + // Header is a net/http/Header structure + Header http.Header + // contains filtered or unexported fields +} +``` +Response is what is returned to a user when they fire off a request + + + + + + + +### func [Delete](/src/target/base.go?s=1221:1283#L22) +``` go +func Delete(url string, ro *RequestOptions) (*Response, error) +``` +Delete takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Get](/src/target/base.go?s=300:359#L1) +``` go +func Get(url string, ro *RequestOptions) (*Response, error) +``` +Get takes 2 parameters and returns a Response Struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Head](/src/target/base.go?s=1841:1901#L38) +``` go +func Head(url string, ro *RequestOptions) (*Response, error) +``` +Head takes 2 parameters and returns a Response channel. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Options](/src/target/base.go?s=2151:2214#L46) +``` go +func Options(url string, ro *RequestOptions) (*Response, error) +``` +Options takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Patch](/src/target/base.go?s=910:971#L14) +``` go +func Patch(url string, ro *RequestOptions) (*Response, error) +``` +Patch takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Post](/src/target/base.go?s=1533:1593#L30) +``` go +func Post(url string, ro *RequestOptions) (*Response, error) +``` +Post takes 2 parameters and returns a Response channel. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + +### func [Put](/src/target/base.go?s=604:663#L6) +``` go +func Put(url string, ro *RequestOptions) (*Response, error) +``` +Put takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil + + + + + +### func (\*Response) [Bytes](/src/target/response.go?s=4298:4331#L169) +``` go +func (r *Response) Bytes() []byte +``` +Bytes returns the response as a byte array + + + + +### func (\*Response) [ClearInternalBuffer](/src/target/response.go?s=4920:4960#L198) +``` go +func (r *Response) ClearInternalBuffer() +``` +ClearInternalBuffer is a function that will clear the internal buffer that we use to hold the .String() and .Bytes() +data. Once you have used these functions – you may want to free up the memory. + + + + +### func (\*Response) [Close](/src/target/response.go?s=1793:1825#L55) +``` go +func (r *Response) Close() error +``` +Close is part of our ability to support io.ReadCloser if someone wants to make use of the raw body + + + + +### func (\*Response) [DownloadToFile](/src/target/response.go?s=2018:2074#L67) +``` go +func (r *Response) DownloadToFile(fileName string) error +``` +DownloadToFile allows you to download the contents of the response to a file + + + + +### func (\*Response) [JSON](/src/target/response.go?s=3300:3353#L124) +``` go +func (r *Response) JSON(userStruct interface{}) error +``` +JSON is a method that will populate a struct that is provided `userStruct` with the JSON returned within the +response body + + + + +### func (\*Response) [Read](/src/target/response.go?s=1551:1603#L45) +``` go +func (r *Response) Read(p []byte) (n int, err error) +``` +Read is part of our ability to support io.ReadCloser if someone wants to make use of the raw body + + + + +### func (\*Response) [String](/src/target/response.go?s=4568:4602#L186) +``` go +func (r *Response) String() string +``` +String returns the response as a string + + + + +### func (\*Response) [XML](/src/target/response.go?s=2792:2874#L101) +``` go +func (r *Response) XML(userStruct interface{}, charsetReader XMLCharDecoder) error +``` +XML is a method that will populate a struct that is provided `userStruct` with the XML returned within the +response body + + + + +## type [Session](/src/target/session.go?s=125:314#L1) +``` go +type Session struct { + // RequestOptions is global options + RequestOptions *RequestOptions + + // HTTPClient is the client that we will use to request the resources + HTTPClient *http.Client +} +``` +Session allows a user to make use of persistent cookies in between +HTTP requests + + + + + + + +### func [NewSession](/src/target/session.go?s=532:576#L8) +``` go +func NewSession(ro *RequestOptions) *Session +``` +NewSession returns a session struct which enables can be used to maintain establish a persistent state with the +server +This function will set UseCookieJar to true as that is the purpose of using the session + + + + + +### func (\*Session) [CloseIdleConnections](/src/target/session.go?s=4738:4778#L124) +``` go +func (s *Session) CloseIdleConnections() +``` +CloseIdleConnections closes the idle connections that a session client may make use of + + + + +### func (\*Session) [Delete](/src/target/session.go?s=3120:3195#L88) +``` go +func (s *Session) Delete(url string, ro *RequestOptions) (*Response, error) +``` +Delete takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Get](/src/target/session.go?s=1776:1848#L58) +``` go +func (s *Session) Get(url string, ro *RequestOptions) (*Response, error) +``` +Get takes 2 parameters and returns a Response Struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Head](/src/target/session.go?s=4022:4095#L108) +``` go +func (s *Session) Head(url string, ro *RequestOptions) (*Response, error) +``` +Head takes 2 parameters and returns a Response channel. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Options](/src/target/session.go?s=4473:4549#L118) +``` go +func (s *Session) Options(url string, ro *RequestOptions) (*Response, error) +``` +Options takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Patch](/src/target/session.go?s=2668:2742#L78) +``` go +func (s *Session) Patch(url string, ro *RequestOptions) (*Response, error) +``` +Patch takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Post](/src/target/session.go?s=3573:3646#L98) +``` go +func (s *Session) Post(url string, ro *RequestOptions) (*Response, error) +``` +Post takes 2 parameters and returns a Response channel. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +### func (\*Session) [Put](/src/target/session.go?s=2221:2293#L68) +``` go +func (s *Session) Put(url string, ro *RequestOptions) (*Response, error) +``` +Put takes 2 parameters and returns a Response struct. These two options are: + + + 1. A URL + 2. A RequestOptions struct + +If you do not intend to use the `RequestOptions` you can just pass nil +A new session is created by calling NewSession with a request options struct + + + + +## type [XMLCharDecoder](/src/target/utils.go?s=1529:1605#L42) +``` go +type XMLCharDecoder func(charset string, input io.Reader) (io.Reader, error) +``` +XMLCharDecoder is a helper type that takes a stream of bytes (not encoded in +UTF-8) and returns a reader that encodes the bytes into UTF-8. This is done +because Go's XML library only supports XML encoded in UTF-8 + + + + + + + + + + + + + + +- - - +Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) diff --git a/vendor/github.com/levigross/grequests/base.go b/vendor/github.com/levigross/grequests/base.go new file mode 100644 index 0000000..748b708 --- /dev/null +++ b/vendor/github.com/levigross/grequests/base.go @@ -0,0 +1,67 @@ +// Package grequests implements a friendly API over Go's existing net/http library +package grequests + +// Get takes 2 parameters and returns a Response Struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Get(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("GET", url, ro) +} + +// Put takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Put(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("PUT", url, ro) +} + +// Patch takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Patch(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("PATCH", url, ro) +} + +// Delete takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Delete(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("DELETE", url, ro) +} + +// Post takes 2 parameters and returns a Response channel. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Post(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("POST", url, ro) +} + +// Head takes 2 parameters and returns a Response channel. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Head(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("HEAD", url, ro) +} + +// Options takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Options(url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest("OPTIONS", url, ro) +} + +// Req takes 3 parameters and returns a Response Struct. These three options are: +// 1. A verb +// 2. A URL +// 3. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +func Req(verb string, url string, ro *RequestOptions) (*Response, error) { + return doRegularRequest(verb, url, ro) +} diff --git a/vendor/github.com/levigross/grequests/file_upload.go b/vendor/github.com/levigross/grequests/file_upload.go new file mode 100644 index 0000000..aba1802 --- /dev/null +++ b/vendor/github.com/levigross/grequests/file_upload.go @@ -0,0 +1,68 @@ +package grequests + +import ( + "errors" + "io" + "os" + "path/filepath" +) + +// FileUpload is a struct that is used to specify the file that a User +// wishes to upload. +type FileUpload struct { + // Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server + FileName string + + // FileContents is happy as long as you pass it a io.ReadCloser (which most file use anyways) + FileContents io.ReadCloser + + // FieldName is form field name + FieldName string + + // FileMime represents which mimetime should be sent along with the file. + // When empty, defaults to application/octet-stream + FileMime string +} + +// FileUploadFromDisk allows you to create a FileUpload struct slice by just specifying a location on the disk +func FileUploadFromDisk(fileName string) ([]FileUpload, error) { + fd, err := os.Open(fileName) + + if err != nil { + return nil, err + } + + return []FileUpload{{FileContents: fd, FileName: fileName}}, nil + +} + +// FileUploadFromGlob allows you to create a FileUpload struct slice by just specifying a glob location on the disk +// this function will gloss over all errors in the files and only upload the files that don't return errors from the glob +func FileUploadFromGlob(fileSystemGlob string) ([]FileUpload, error) { + files, err := filepath.Glob(fileSystemGlob) + + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, errors.New("grequests: No files have been returned in the glob") + } + + filesToUpload := make([]FileUpload, 0, len(files)) + + for _, f := range files { + if s, err := os.Stat(f); err != nil || s.IsDir() { + continue + } + + // ignoring error because I can stat the file + fd, _ := os.Open(f) + + filesToUpload = append(filesToUpload, FileUpload{FileContents: fd, FileName: filepath.Base(fd.Name())}) + + } + + return filesToUpload, nil + +} diff --git a/vendor/github.com/levigross/grequests/request.go b/vendor/github.com/levigross/grequests/request.go new file mode 100644 index 0000000..f17566c --- /dev/null +++ b/vendor/github.com/levigross/grequests/request.go @@ -0,0 +1,585 @@ +package grequests + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net" + "net/http" + "net/http/cookiejar" + "net/textproto" + "net/url" + "strconv" + "strings" + "time" + + "github.com/google/go-querystring/query" + + "context" + + "golang.org/x/net/publicsuffix" +) + +// RequestOptions is the location that of where the data +type RequestOptions struct { + + // Data is a map of key values that will eventually convert into the + // query string of a GET request or the body of a POST request. + Data map[string]string + + // Params is a map of query strings that may be used within a GET request + Params map[string]string + + // QueryStruct is a struct that encapsulates a set of URL query params + // this paramter is mutually exclusive with `Params map[string]string` (they cannot be combined) + // for more information please see https://godoc.org/github.com/google/go-querystring/query + QueryStruct interface{} + + // Files is where you can include files to upload. The use of this data + // structure is limited to POST requests + Files []FileUpload + + // JSON can be used when you wish to send JSON within the request body + JSON interface{} + + // XML can be used if you wish to send XML within the request body + XML interface{} + + // Headers if you want to add custom HTTP headers to the request, + // this is your friend + Headers map[string]string + + // InsecureSkipVerify is a flag that specifies if we should validate the + // server's TLS certificate. It should be noted that Go's TLS verify mechanism + // doesn't validate if a certificate has been revoked + InsecureSkipVerify bool + + // DisableCompression will disable gzip compression on requests + DisableCompression bool + + // UserAgent allows you to set an arbitrary custom user agent + UserAgent string + + // Host allows you to set an arbitrary custom host + Host string + + // Auth allows you to specify a user name and password that you wish to + // use when requesting the URL. It will use basic HTTP authentication + // formatting the username and password in base64 the format is: + // []string{username, password} + Auth []string + + // IsAjax is a flag that can be set to make the request appear + // to be generated by browser Javascript + IsAjax bool + + // Cookies is an array of `http.Cookie` that allows you to attach + // cookies to your request + Cookies []*http.Cookie + + // UseCookieJar will create a custom HTTP client that will + // process and store HTTP cookies when they are sent down + UseCookieJar bool + + // Proxies is a map in the following format + // *protocol* => proxy address e.g http => http://127.0.0.1:8080 + Proxies map[string]*url.URL + + // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // wait for a TLS handshake. Zero means no timeout. + TLSHandshakeTimeout time.Duration + + // DialTimeout is the maximum amount of time a dial will wait for + // a connect to complete. + DialTimeout time.Duration + + // KeepAlive specifies the keep-alive period for an active + // network connection. If zero, keep-alive are not enabled. + DialKeepAlive time.Duration + + // RequestTimeout is the maximum amount of time a whole request(include dial / request / redirect) + // will wait. + RequestTimeout time.Duration + + // HTTPClient can be provided if you wish to supply a custom HTTP client + // this is useful if you want to use an OAUTH client with your request. + HTTPClient *http.Client + + // SensitiveHTTPHeaders is a map of sensitive HTTP headers that a user + // doesn't want passed on a redirect. + SensitiveHTTPHeaders map[string]struct{} + + // RedirectLimit is the acceptable amount of redirects that we should expect + // before returning an error be default this is set to 30. You can change this + // globally by modifying the `RedirectLimit` variable. + RedirectLimit int + + // RequestBody allows you to put anything matching an `io.Reader` into the request + // this option will take precedence over any other request option specified + RequestBody io.Reader + + // CookieJar allows you to specify a special cookiejar to use with your request. + // this option will take precedence over the `UseCookieJar` option above. + CookieJar http.CookieJar + + // Context can be used to maintain state between requests https://golang.org/pkg/context/#Context + Context context.Context +} + +func doRegularRequest(requestVerb, url string, ro *RequestOptions) (*Response, error) { + return buildResponse(buildRequest(requestVerb, url, ro, nil)) +} + +func doSessionRequest(requestVerb, url string, ro *RequestOptions, httpClient *http.Client) (*Response, error) { + return buildResponse(buildRequest(requestVerb, url, ro, httpClient)) +} + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +// buildRequest is where most of the magic happens for request processing +func buildRequest(httpMethod, url string, ro *RequestOptions, httpClient *http.Client) (*http.Response, error) { + if ro == nil { + ro = &RequestOptions{} + } + + if ro.CookieJar != nil { + ro.UseCookieJar = true + } + + // Create our own HTTP client + + if httpClient == nil { + httpClient = BuildHTTPClient(*ro) + } + + var err error // we don't want to shadow url so we won't use := + switch { + case len(ro.Params) != 0: + if url, err = buildURLParams(url, ro.Params); err != nil { + return nil, err + } + case ro.QueryStruct != nil: + if url, err = buildURLStruct(url, ro.QueryStruct); err != nil { + return nil, err + } + } + + // Build the request + req, err := buildHTTPRequest(httpMethod, url, ro) + + if err != nil { + return nil, err + } + + // Do we need to add any HTTP headers or Basic Auth? + addHTTPHeaders(ro, req) + addCookies(ro, req) + + addRedirectFunctionality(httpClient, ro) + + if ro.Context != nil { + req = req.WithContext(ro.Context) + } + + return httpClient.Do(req) +} + +func buildHTTPRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + if ro.RequestBody != nil { + return http.NewRequest(httpMethod, userURL, ro.RequestBody) + } + + if ro.JSON != nil { + return createBasicJSONRequest(httpMethod, userURL, ro) + } + + if ro.XML != nil { + return createBasicXMLRequest(httpMethod, userURL, ro) + } + + if ro.Files != nil { + return createFileUploadRequest(httpMethod, userURL, ro) + } + + if ro.Data != nil { + return createBasicRequest(httpMethod, userURL, ro) + } + + return http.NewRequest(httpMethod, userURL, nil) +} + +func createFileUploadRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + if httpMethod == "POST" { + return createMultiPartPostRequest(httpMethod, userURL, ro) + } + + // This may be a PUT or PATCH request so we will just put the raw + // io.ReadCloser in the request body + // and guess the MIME type from the file name + + // At the moment, we will only support 1 file upload as a time + // when uploading using PUT or PATCH + + req, err := http.NewRequest(httpMethod, userURL, ro.Files[0].FileContents) + + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", mime.TypeByExtension(ro.Files[0].FileName)) + + return req, nil + +} + +func createBasicXMLRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + var reader io.Reader + + switch ro.XML.(type) { + case string: + reader = strings.NewReader(ro.XML.(string)) + case []byte: + reader = bytes.NewReader(ro.XML.([]byte)) + default: + byteSlice, err := xml.Marshal(ro.XML) + if err != nil { + return nil, err + } + reader = bytes.NewReader(byteSlice) + } + + req, err := http.NewRequest(httpMethod, userURL, reader) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/xml") + + return req, nil + +} +func createMultiPartPostRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + requestBody := &bytes.Buffer{} + + multipartWriter := multipart.NewWriter(requestBody) + + for i, f := range ro.Files { + + if f.FileContents == nil { + return nil, errors.New("grequests: Pointer FileContents cannot be nil") + } + + fieldName := f.FieldName + + if fieldName == "" { + if len(ro.Files) > 1 { + fieldName = strings.Join([]string{"file", strconv.Itoa(i + 1)}, "") + } else { + fieldName = "file" + } + } + + var writer io.Writer + var err error + + if f.FileMime != "" { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(fieldName), escapeQuotes(f.FileName))) + h.Set("Content-Type", f.FileMime) + writer, err = multipartWriter.CreatePart(h) + } else { + writer, err = multipartWriter.CreateFormFile(fieldName, f.FileName) + } + + if err != nil { + return nil, err + } + + if _, err = io.Copy(writer, f.FileContents); err != nil && err != io.EOF { + return nil, err + } + + f.FileContents.Close() + + } + + // Populate the other parts of the form (if there are any) + for key, value := range ro.Data { + multipartWriter.WriteField(key, value) + } + + if err := multipartWriter.Close(); err != nil { + return nil, err + } + + req, err := http.NewRequest(httpMethod, userURL, requestBody) + + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", multipartWriter.FormDataContentType()) + + return req, err +} + +func createBasicJSONRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + + var reader io.Reader + switch ro.JSON.(type) { + case string: + reader = strings.NewReader(ro.JSON.(string)) + case []byte: + reader = bytes.NewReader(ro.JSON.([]byte)) + default: + byteSlice, err := json.Marshal(ro.JSON) + if err != nil { + return nil, err + } + reader = bytes.NewReader(byteSlice) + } + + req, err := http.NewRequest(httpMethod, userURL, reader) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + + return req, nil + +} +func createBasicRequest(httpMethod, userURL string, ro *RequestOptions) (*http.Request, error) { + + req, err := http.NewRequest(httpMethod, userURL, strings.NewReader(encodePostValues(ro.Data))) + + if err != nil { + return nil, err + } + + // The content type must be set to a regular form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + return req, nil +} + +func encodePostValues(postValues map[string]string) string { + urlValues := &url.Values{} + + for key, value := range postValues { + urlValues.Set(key, value) + } + + return urlValues.Encode() // This will sort all of the string values +} + +// proxySettings will default to the default proxy settings if none are provided +// if settings are provided – they will override the environment variables +func (ro RequestOptions) proxySettings(req *http.Request) (*url.URL, error) { + // No proxies – lets use the default + if len(ro.Proxies) == 0 { + return http.ProxyFromEnvironment(req) + } + + // There was a proxy specified – do we support the protocol? + if _, ok := ro.Proxies[req.URL.Scheme]; ok { + return ro.Proxies[req.URL.Scheme], nil + } + + // Proxies were specified but not for any protocol that we use + return http.ProxyFromEnvironment(req) + +} + +// dontUseDefaultClient will tell the "client creator" if a custom client is needed +// it checks the following items (and will create a custom client of these are) +// true +// 1. Do we want to accept invalid SSL certificates? +// 2. Do we want to disable compression? +// 3. Do we want a custom proxy? +// 4. Do we want to change the default timeout for TLS Handshake? +// 5. Do we want to change the default request timeout? +// 6. Do we want to change the default connection timeout? +// 7. Do you want to use the http.Client's cookieJar? +func (ro RequestOptions) dontUseDefaultClient() bool { + switch { + case ro.InsecureSkipVerify == true: + case ro.DisableCompression == true: + case len(ro.Proxies) != 0: + case ro.TLSHandshakeTimeout != 0: + case ro.DialTimeout != 0: + case ro.DialKeepAlive != 0: + case len(ro.Cookies) != 0: + case ro.UseCookieJar != false: + case ro.RequestTimeout != 0: + default: + return false + } + return true +} + +// BuildHTTPClient is a function that will return a custom HTTP client based on the request options provided +// the check is in UseDefaultClient +func BuildHTTPClient(ro RequestOptions) *http.Client { + + if ro.HTTPClient != nil { + return ro.HTTPClient + } + + // Does the user want to change the defaults? + if !ro.dontUseDefaultClient() { + return http.DefaultClient + } + + // Using the user config for tls timeout or default + if ro.TLSHandshakeTimeout == 0 { + ro.TLSHandshakeTimeout = tlsHandshakeTimeout + } + + // Using the user config for dial timeout or default + if ro.DialTimeout == 0 { + ro.DialTimeout = dialTimeout + } + + // Using the user config for dial keep alive or default + if ro.DialKeepAlive == 0 { + ro.DialKeepAlive = dialKeepAlive + } + + if ro.RequestTimeout == 0 { + ro.RequestTimeout = requestTimeout + } + + var cookieJar http.CookieJar + + if ro.UseCookieJar { + if ro.CookieJar != nil { + cookieJar = ro.CookieJar + } else { + // The function does not return an error ever... so we are just ignoring it + cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + } + } + + return &http.Client{ + Jar: cookieJar, + Transport: createHTTPTransport(ro), + Timeout: ro.RequestTimeout, + } +} + +func createHTTPTransport(ro RequestOptions) *http.Transport { + ourHTTPTransport := &http.Transport{ + // These are borrowed from the default transporter + Proxy: ro.proxySettings, + Dial: (&net.Dialer{ + Timeout: ro.DialTimeout, + KeepAlive: ro.DialKeepAlive, + }).Dial, + TLSHandshakeTimeout: ro.TLSHandshakeTimeout, + + // Here comes the user settings + TLSClientConfig: &tls.Config{InsecureSkipVerify: ro.InsecureSkipVerify}, + DisableCompression: ro.DisableCompression, + } + EnsureTransporterFinalized(ourHTTPTransport) + return ourHTTPTransport +} + +// buildURLParams returns a URL with all of the params +// Note: This function will override current URL params if they contradict what is provided in the map +// That is what the "magic" is on the last line +func buildURLParams(userURL string, params map[string]string) (string, error) { + parsedURL, err := url.Parse(userURL) + + if err != nil { + return "", err + } + + parsedQuery, err := url.ParseQuery(parsedURL.RawQuery) + + if err != nil { + return "", nil + } + + for key, value := range params { + parsedQuery.Set(key, value) + } + + return addQueryParams(parsedURL, parsedQuery), nil +} + +// addHTTPHeaders adds any additional HTTP headers that need to be added are added here including: +// 1. Custom User agent +// 2. Authorization Headers +// 3. Any other header requested +func addHTTPHeaders(ro *RequestOptions, req *http.Request) { + for key, value := range ro.Headers { + req.Header.Set(key, value) + } + + if ro.UserAgent != "" { + req.Header.Set("User-Agent", ro.UserAgent) + } else { + req.Header.Set("User-Agent", localUserAgent) + } + + if ro.Host != "" { + req.Host = ro.Host + } + + if ro.Auth != nil { + req.SetBasicAuth(ro.Auth[0], ro.Auth[1]) + } + + if ro.IsAjax == true { + req.Header.Set("X-Requested-With", "XMLHttpRequest") + } +} + +func addCookies(ro *RequestOptions, req *http.Request) { + for _, c := range ro.Cookies { + req.AddCookie(c) + } +} + +func addQueryParams(parsedURL *url.URL, parsedQuery url.Values) string { + return strings.Join([]string{strings.Replace(parsedURL.String(), "?"+parsedURL.RawQuery, "", -1), parsedQuery.Encode()}, "?") +} + +func buildURLStruct(userURL string, URLStruct interface{}) (string, error) { + parsedURL, err := url.Parse(userURL) + + if err != nil { + return "", err + } + + parsedQuery, err := url.ParseQuery(parsedURL.RawQuery) + + if err != nil { + return "", err + } + + queryStruct, err := query.Values(URLStruct) + if err != nil { + return "", err + } + + for key, value := range queryStruct { + for _, v := range value { + parsedQuery.Add(key, v) + } + } + + return addQueryParams(parsedURL, parsedQuery), nil +} diff --git a/vendor/github.com/levigross/grequests/response.go b/vendor/github.com/levigross/grequests/response.go new file mode 100644 index 0000000..e2fce42 --- /dev/null +++ b/vendor/github.com/levigross/grequests/response.go @@ -0,0 +1,215 @@ +package grequests + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "io" + "io/ioutil" + "net/http" + "os" +) + +// Response is what is returned to a user when they fire off a request +type Response struct { + + // Ok is a boolean flag that validates that the server returned a 2xx code + Ok bool + + // This is the Go error flag – if something went wrong within the request, this flag will be set. + Error error + + // We want to abstract (at least at the moment) the Go http.Response object away. So we are going to make use of it + // internal but not give the user access + RawResponse *http.Response + + // StatusCode is the HTTP Status Code returned by the HTTP Response. Taken from resp.StatusCode + StatusCode int + + // Header is a net/http/Header structure + Header http.Header + + internalByteBuffer *bytes.Buffer +} + +func buildResponse(resp *http.Response, err error) (*Response, error) { + // If the connection didn't succeed we just return a blank response + if err != nil { + return &Response{Error: err}, err + } + + goodResp := &Response{ + // If your code is within the 2xx range – the response is considered `Ok` + Ok: resp.StatusCode >= 200 && resp.StatusCode < 300, + Error: nil, + RawResponse: resp, + StatusCode: resp.StatusCode, + Header: resp.Header, + internalByteBuffer: bytes.NewBuffer([]byte{}), + } + // EnsureResponseFinalized(goodResp) This will come back in 1.0 + return goodResp, nil +} + +// Read is part of our ability to support io.ReadCloser if someone wants to make use of the raw body +func (r *Response) Read(p []byte) (n int, err error) { + + if r.Error != nil { + return -1, r.Error + } + + return r.RawResponse.Body.Read(p) +} + +// Close is part of our ability to support io.ReadCloser if someone wants to make use of the raw body +func (r *Response) Close() error { + + if r.Error != nil { + return r.Error + } + + io.Copy(ioutil.Discard, r) + + return r.RawResponse.Body.Close() +} + +// DownloadToFile allows you to download the contents of the response to a file +func (r *Response) DownloadToFile(fileName string) error { + + if r.Error != nil { + return r.Error + } + + fd, err := os.Create(fileName) + + if err != nil { + return err + } + + defer r.Close() // This is a noop if we use the internal ByteBuffer + defer fd.Close() + + if _, err := io.Copy(fd, r.getInternalReader()); err != nil && err != io.EOF { + return err + } + + return nil +} + +// getInternalReader because we implement io.ReadCloser and optionally hold a large buffer of the response (created by +// the user's request) +func (r *Response) getInternalReader() io.Reader { + + if r.internalByteBuffer.Len() != 0 { + return r.internalByteBuffer + } + return r +} + +// XML is a method that will populate a struct that is provided `userStruct` with the XML returned within the +// response body +func (r *Response) XML(userStruct interface{}, charsetReader XMLCharDecoder) error { + + if r.Error != nil { + return r.Error + } + + xmlDecoder := xml.NewDecoder(r.getInternalReader()) + + if charsetReader != nil { + xmlDecoder.CharsetReader = charsetReader + } + + defer r.Close() + + if err := xmlDecoder.Decode(&userStruct); err != nil && err != io.EOF { + return err + } + + return nil +} + +// JSON is a method that will populate a struct that is provided `userStruct` with the JSON returned within the +// response body +func (r *Response) JSON(userStruct interface{}) error { + + if r.Error != nil { + return r.Error + } + + jsonDecoder := json.NewDecoder(r.getInternalReader()) + defer r.Close() + + if err := jsonDecoder.Decode(&userStruct); err != nil && err != io.EOF { + return err + } + + return nil +} + +// createResponseBytesBuffer is a utility method that will populate the internal byte reader – this is largely used for .String() +// and .Bytes() +func (r *Response) populateResponseByteBuffer() { + + // Have I done this already? + if r.internalByteBuffer.Len() != 0 { + return + } + + defer r.Close() + + // Is there any content? + if r.RawResponse.ContentLength == 0 { + return + } + + // Did the server tell us how big the response is going to be? + if r.RawResponse.ContentLength > 0 { + r.internalByteBuffer.Grow(int(r.RawResponse.ContentLength)) + } + + if _, err := io.Copy(r.internalByteBuffer, r); err != nil && err != io.EOF { + r.Error = err + r.RawResponse.Body.Close() + } + +} + +// Bytes returns the response as a byte array +func (r *Response) Bytes() []byte { + + if r.Error != nil { + return nil + } + + r.populateResponseByteBuffer() + + // Are we still empty? + if r.internalByteBuffer.Len() == 0 { + return nil + } + return r.internalByteBuffer.Bytes() + +} + +// String returns the response as a string +func (r *Response) String() string { + if r.Error != nil { + return "" + } + + r.populateResponseByteBuffer() + + return r.internalByteBuffer.String() +} + +// ClearInternalBuffer is a function that will clear the internal buffer that we use to hold the .String() and .Bytes() +// data. Once you have used these functions – you may want to free up the memory. +func (r *Response) ClearInternalBuffer() { + + if r == nil || r.internalByteBuffer == nil { + return + } + + r.internalByteBuffer.Reset() +} diff --git a/vendor/github.com/levigross/grequests/session.go b/vendor/github.com/levigross/grequests/session.go new file mode 100644 index 0000000..80dd832 --- /dev/null +++ b/vendor/github.com/levigross/grequests/session.go @@ -0,0 +1,136 @@ +package grequests + +import "net/http" + +// Session allows a user to make use of persistent cookies in between +// HTTP requests +type Session struct { + // RequestOptions is global options + RequestOptions *RequestOptions + + // HTTPClient is the client that we will use to request the resources + HTTPClient *http.Client +} + +// NewSession returns a session struct which enables can be used to maintain establish a persistent state with the +// server +// This function will set UseCookieJar to true as that is the purpose of using the session +func NewSession(ro *RequestOptions) *Session { + if ro == nil { + ro = &RequestOptions{} + } + + ro.UseCookieJar = true + + return &Session{RequestOptions: ro, HTTPClient: BuildHTTPClient(*ro)} +} + +// Combine session options and request options +// 1. UserAgent +// 2. Host +// 3. Auth +// 4. Headers +func (s *Session) combineRequestOptions(ro *RequestOptions) *RequestOptions { + if ro == nil { + ro = &RequestOptions{} + } + + if ro.UserAgent == "" && s.RequestOptions.UserAgent != "" { + ro.UserAgent = s.RequestOptions.UserAgent + } + + if ro.Host == "" && s.RequestOptions.Host != "" { + ro.Host = s.RequestOptions.Host + } + + if ro.Auth == nil && s.RequestOptions.Auth != nil { + ro.Auth = s.RequestOptions.Auth + } + + if len(s.RequestOptions.Headers) > 0 || len(ro.Headers) > 0 { + headers := make(map[string]string) + for k, v := range s.RequestOptions.Headers { + headers[k] = v + } + for k, v := range ro.Headers { + headers[k] = v + } + ro.Headers = headers + } + return ro +} + +// Get takes 2 parameters and returns a Response Struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Get(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("GET", url, ro, s.HTTPClient) +} + +// Put takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Put(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("PUT", url, ro, s.HTTPClient) +} + +// Patch takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Patch(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("PATCH", url, ro, s.HTTPClient) +} + +// Delete takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Delete(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("DELETE", url, ro, s.HTTPClient) +} + +// Post takes 2 parameters and returns a Response channel. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Post(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("POST", url, ro, s.HTTPClient) +} + +// Head takes 2 parameters and returns a Response channel. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Head(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("HEAD", url, ro, s.HTTPClient) +} + +// Options takes 2 parameters and returns a Response struct. These two options are: +// 1. A URL +// 2. A RequestOptions struct +// If you do not intend to use the `RequestOptions` you can just pass nil +// A new session is created by calling NewSession with a request options struct +func (s *Session) Options(url string, ro *RequestOptions) (*Response, error) { + ro = s.combineRequestOptions(ro) + return doSessionRequest("OPTIONS", url, ro, s.HTTPClient) +} + +// CloseIdleConnections closes the idle connections that a session client may make use of +func (s *Session) CloseIdleConnections() { + s.HTTPClient.Transport.(*http.Transport).CloseIdleConnections() +} diff --git a/vendor/github.com/levigross/grequests/utils.go b/vendor/github.com/levigross/grequests/utils.go new file mode 100644 index 0000000..4445837 --- /dev/null +++ b/vendor/github.com/levigross/grequests/utils.go @@ -0,0 +1,102 @@ +package grequests + +import ( + "errors" + "io" + "net/http" + "runtime" + "time" +) + +const ( + localUserAgent = "GRequests/0.10" + + // Default value for net.Dialer Timeout + dialTimeout = 30 * time.Second + + // Default value for net.Dialer KeepAlive + dialKeepAlive = 30 * time.Second + + // Default value for http.Transport TLSHandshakeTimeout + tlsHandshakeTimeout = 10 * time.Second + + // Default value for Request Timeout + requestTimeout = 90 * time.Second +) + +var ( + // ErrRedirectLimitExceeded is the error returned when the request responded + // with too many redirects + ErrRedirectLimitExceeded = errors.New("grequests: Request exceeded redirect count") + + // RedirectLimit is a tunable variable that specifies how many times we can + // redirect in response to a redirect. This is the global variable, if you + // wish to set this on a request by request basis, set it within the + // `RequestOptions` structure + RedirectLimit = 30 + + // SensitiveHTTPHeaders is a map of sensitive HTTP headers that a user + // doesn't want passed on a redirect. This is the global variable, if you + // wish to set this on a request by request basis, set it within the + // `RequestOptions` structure + SensitiveHTTPHeaders = map[string]struct{}{ + "Www-Authenticate": {}, + "Authorization": {}, + "Proxy-Authorization": {}, + } +) + +// XMLCharDecoder is a helper type that takes a stream of bytes (not encoded in +// UTF-8) and returns a reader that encodes the bytes into UTF-8. This is done +// because Go's XML library only supports XML encoded in UTF-8 +type XMLCharDecoder func(charset string, input io.Reader) (io.Reader, error) + +func addRedirectFunctionality(client *http.Client, ro *RequestOptions) { + if client.CheckRedirect != nil { + return + } + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if ro.RedirectLimit == 0 { + ro.RedirectLimit = RedirectLimit + } + + if len(via) >= ro.RedirectLimit { + return ErrRedirectLimitExceeded + } + + if ro.SensitiveHTTPHeaders == nil { + ro.SensitiveHTTPHeaders = SensitiveHTTPHeaders + } + + for k, vv := range via[0].Header { + // Is this a sensitive header? + if _, found := ro.SensitiveHTTPHeaders[k]; found { + continue + } + + for _, v := range vv { + req.Header.Add(k, v) + } + } + + return nil + } +} + +// EnsureTransporterFinalized will ensure that when the HTTP client is GCed +// the runtime will close the idle connections (so that they won't leak) +// this function was adopted from Hashicorp's go-cleanhttp package +func EnsureTransporterFinalized(httpTransport *http.Transport) { + runtime.SetFinalizer(&httpTransport, func(transportInt **http.Transport) { + (*transportInt).CloseIdleConnections() + }) +} + +// EnsureResponseFinalized will ensure that when the Response is GCed +// the request body is closed so we aren't leaking fds +// func EnsureResponseFinalized(httpResp *Response) { +// runtime.SetFinalizer(&httpResp, func(httpResponseInt **Response) { +// (*httpResponseInt).RawResponse.Body.Close() +// }) +// } +// This will come back in 1.0 diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/golang.org/x/net/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/golang.org/x/net/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/net/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/net/publicsuffix/gen.go b/vendor/golang.org/x/net/publicsuffix/gen.go new file mode 100644 index 0000000..f85a3c3 --- /dev/null +++ b/vendor/golang.org/x/net/publicsuffix/gen.go @@ -0,0 +1,713 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates table.go and table_test.go based on the authoritative +// public suffix list at https://publicsuffix.org/list/effective_tld_names.dat +// +// The version is derived from +// https://api.github.com/repos/publicsuffix/list/commits?path=public_suffix_list.dat +// and a human-readable form is at +// https://github.com/publicsuffix/list/commits/master/public_suffix_list.dat +// +// To fetch a particular git revision, such as 5c70ccd250, pass +// -url "https://raw.githubusercontent.com/publicsuffix/list/5c70ccd250/public_suffix_list.dat" +// and -version "an explicit version string". + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "sort" + "strings" + + "golang.org/x/net/idna" +) + +const ( + // These sum of these four values must be no greater than 32. + nodesBitsChildren = 10 + nodesBitsICANN = 1 + nodesBitsTextOffset = 15 + nodesBitsTextLength = 6 + + // These sum of these four values must be no greater than 32. + childrenBitsWildcard = 1 + childrenBitsNodeType = 2 + childrenBitsHi = 14 + childrenBitsLo = 14 +) + +var ( + maxChildren int + maxTextOffset int + maxTextLength int + maxHi uint32 + maxLo uint32 +) + +func max(a, b int) int { + if a < b { + return b + } + return a +} + +func u32max(a, b uint32) uint32 { + if a < b { + return b + } + return a +} + +const ( + nodeTypeNormal = 0 + nodeTypeException = 1 + nodeTypeParentOnly = 2 + numNodeType = 3 +) + +func nodeTypeStr(n int) string { + switch n { + case nodeTypeNormal: + return "+" + case nodeTypeException: + return "!" + case nodeTypeParentOnly: + return "o" + } + panic("unreachable") +} + +const ( + defaultURL = "https://publicsuffix.org/list/effective_tld_names.dat" + gitCommitURL = "https://api.github.com/repos/publicsuffix/list/commits?path=public_suffix_list.dat" +) + +var ( + labelEncoding = map[string]uint32{} + labelsList = []string{} + labelsMap = map[string]bool{} + rules = []string{} + + // validSuffixRE is used to check that the entries in the public suffix + // list are in canonical form (after Punycode encoding). Specifically, + // capital letters are not allowed. + validSuffixRE = regexp.MustCompile(`^[a-z0-9_\!\*\-\.]+$`) + + shaRE = regexp.MustCompile(`"sha":"([^"]+)"`) + dateRE = regexp.MustCompile(`"committer":{[^{]+"date":"([^"]+)"`) + + comments = flag.Bool("comments", false, "generate table.go comments, for debugging") + subset = flag.Bool("subset", false, "generate only a subset of the full table, for debugging") + url = flag.String("url", defaultURL, "URL of the publicsuffix.org list. If empty, stdin is read instead") + v = flag.Bool("v", false, "verbose output (to stderr)") + version = flag.String("version", "", "the effective_tld_names.dat version") +) + +func main() { + if err := main1(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func main1() error { + flag.Parse() + if nodesBitsTextLength+nodesBitsTextOffset+nodesBitsICANN+nodesBitsChildren > 32 { + return fmt.Errorf("not enough bits to encode the nodes table") + } + if childrenBitsLo+childrenBitsHi+childrenBitsNodeType+childrenBitsWildcard > 32 { + return fmt.Errorf("not enough bits to encode the children table") + } + if *version == "" { + if *url != defaultURL { + return fmt.Errorf("-version was not specified, and the -url is not the default one") + } + sha, date, err := gitCommit() + if err != nil { + return err + } + *version = fmt.Sprintf("publicsuffix.org's public_suffix_list.dat, git revision %s (%s)", sha, date) + } + var r io.Reader = os.Stdin + if *url != "" { + res, err := http.Get(*url) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("bad GET status for %s: %d", *url, res.Status) + } + r = res.Body + defer res.Body.Close() + } + + var root node + icann := false + br := bufio.NewReader(r) + for { + s, err := br.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return err + } + s = strings.TrimSpace(s) + if strings.Contains(s, "BEGIN ICANN DOMAINS") { + icann = true + continue + } + if strings.Contains(s, "END ICANN DOMAINS") { + icann = false + continue + } + if s == "" || strings.HasPrefix(s, "//") { + continue + } + s, err = idna.ToASCII(s) + if err != nil { + return err + } + if !validSuffixRE.MatchString(s) { + return fmt.Errorf("bad publicsuffix.org list data: %q", s) + } + + if *subset { + switch { + case s == "ac.jp" || strings.HasSuffix(s, ".ac.jp"): + case s == "ak.us" || strings.HasSuffix(s, ".ak.us"): + case s == "ao" || strings.HasSuffix(s, ".ao"): + case s == "ar" || strings.HasSuffix(s, ".ar"): + case s == "arpa" || strings.HasSuffix(s, ".arpa"): + case s == "cy" || strings.HasSuffix(s, ".cy"): + case s == "dyndns.org" || strings.HasSuffix(s, ".dyndns.org"): + case s == "jp": + case s == "kobe.jp" || strings.HasSuffix(s, ".kobe.jp"): + case s == "kyoto.jp" || strings.HasSuffix(s, ".kyoto.jp"): + case s == "om" || strings.HasSuffix(s, ".om"): + case s == "uk" || strings.HasSuffix(s, ".uk"): + case s == "uk.com" || strings.HasSuffix(s, ".uk.com"): + case s == "tw" || strings.HasSuffix(s, ".tw"): + case s == "zw" || strings.HasSuffix(s, ".zw"): + case s == "xn--p1ai" || strings.HasSuffix(s, ".xn--p1ai"): + // xn--p1ai is Russian-Cyrillic "рф". + default: + continue + } + } + + rules = append(rules, s) + + nt, wildcard := nodeTypeNormal, false + switch { + case strings.HasPrefix(s, "*."): + s, nt = s[2:], nodeTypeParentOnly + wildcard = true + case strings.HasPrefix(s, "!"): + s, nt = s[1:], nodeTypeException + } + labels := strings.Split(s, ".") + for n, i := &root, len(labels)-1; i >= 0; i-- { + label := labels[i] + n = n.child(label) + if i == 0 { + if nt != nodeTypeParentOnly && n.nodeType == nodeTypeParentOnly { + n.nodeType = nt + } + n.icann = n.icann && icann + n.wildcard = n.wildcard || wildcard + } + labelsMap[label] = true + } + } + labelsList = make([]string, 0, len(labelsMap)) + for label := range labelsMap { + labelsList = append(labelsList, label) + } + sort.Strings(labelsList) + + if err := generate(printReal, &root, "table.go"); err != nil { + return err + } + if err := generate(printTest, &root, "table_test.go"); err != nil { + return err + } + return nil +} + +func generate(p func(io.Writer, *node) error, root *node, filename string) error { + buf := new(bytes.Buffer) + if err := p(buf, root); err != nil { + return err + } + b, err := format.Source(buf.Bytes()) + if err != nil { + return err + } + return ioutil.WriteFile(filename, b, 0644) +} + +func gitCommit() (sha, date string, retErr error) { + res, err := http.Get(gitCommitURL) + if err != nil { + return "", "", err + } + if res.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("bad GET status for %s: %d", gitCommitURL, res.Status) + } + defer res.Body.Close() + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + if m := shaRE.FindSubmatch(b); m != nil { + sha = string(m[1]) + } + if m := dateRE.FindSubmatch(b); m != nil { + date = string(m[1]) + } + if sha == "" || date == "" { + retErr = fmt.Errorf("could not find commit SHA and date in %s", gitCommitURL) + } + return sha, date, retErr +} + +func printTest(w io.Writer, n *node) error { + fmt.Fprintf(w, "// generated by go run gen.go; DO NOT EDIT\n\n") + fmt.Fprintf(w, "package publicsuffix\n\nvar rules = [...]string{\n") + for _, rule := range rules { + fmt.Fprintf(w, "%q,\n", rule) + } + fmt.Fprintf(w, "}\n\nvar nodeLabels = [...]string{\n") + if err := n.walk(w, printNodeLabel); err != nil { + return err + } + fmt.Fprintf(w, "}\n") + return nil +} + +func printReal(w io.Writer, n *node) error { + const header = `// generated by go run gen.go; DO NOT EDIT + +package publicsuffix + +const version = %q + +const ( + nodesBitsChildren = %d + nodesBitsICANN = %d + nodesBitsTextOffset = %d + nodesBitsTextLength = %d + + childrenBitsWildcard = %d + childrenBitsNodeType = %d + childrenBitsHi = %d + childrenBitsLo = %d +) + +const ( + nodeTypeNormal = %d + nodeTypeException = %d + nodeTypeParentOnly = %d +) + +// numTLD is the number of top level domains. +const numTLD = %d + +` + fmt.Fprintf(w, header, *version, + nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength, + childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo, + nodeTypeNormal, nodeTypeException, nodeTypeParentOnly, len(n.children)) + + text := combineText(labelsList) + if text == "" { + return fmt.Errorf("internal error: makeText returned no text") + } + for _, label := range labelsList { + offset, length := strings.Index(text, label), len(label) + if offset < 0 { + return fmt.Errorf("internal error: could not find %q in text %q", label, text) + } + maxTextOffset, maxTextLength = max(maxTextOffset, offset), max(maxTextLength, length) + if offset >= 1<= 1< 64 { + n, plus = 64, " +" + } + fmt.Fprintf(w, "%q%s\n", text[:n], plus) + text = text[n:] + } + + if err := n.walk(w, assignIndexes); err != nil { + return err + } + + fmt.Fprintf(w, ` + +// nodes is the list of nodes. Each node is represented as a uint32, which +// encodes the node's children, wildcard bit and node type (as an index into +// the children array), ICANN bit and text. +// +// If the table was generated with the -comments flag, there is a //-comment +// after each node's data. In it is the nodes-array indexes of the children, +// formatted as (n0x1234-n0x1256), with * denoting the wildcard bit. The +// nodeType is printed as + for normal, ! for exception, and o for parent-only +// nodes that have children but don't match a domain label in their own right. +// An I denotes an ICANN domain. +// +// The layout within the uint32, from MSB to LSB, is: +// [%2d bits] unused +// [%2d bits] children index +// [%2d bits] ICANN bit +// [%2d bits] text index +// [%2d bits] text length +var nodes = [...]uint32{ +`, + 32-nodesBitsChildren-nodesBitsICANN-nodesBitsTextOffset-nodesBitsTextLength, + nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength) + if err := n.walk(w, printNode); err != nil { + return err + } + fmt.Fprintf(w, `} + +// children is the list of nodes' children, the parent's wildcard bit and the +// parent's node type. If a node has no children then their children index +// will be in the range [0, 6), depending on the wildcard bit and node type. +// +// The layout within the uint32, from MSB to LSB, is: +// [%2d bits] unused +// [%2d bits] wildcard bit +// [%2d bits] node type +// [%2d bits] high nodes index (exclusive) of children +// [%2d bits] low nodes index (inclusive) of children +var children=[...]uint32{ +`, + 32-childrenBitsWildcard-childrenBitsNodeType-childrenBitsHi-childrenBitsLo, + childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo) + for i, c := range childrenEncoding { + s := "---------------" + lo := c & (1<> childrenBitsLo) & (1<>(childrenBitsLo+childrenBitsHi)) & (1<>(childrenBitsLo+childrenBitsHi+childrenBitsNodeType) != 0 + if *comments { + fmt.Fprintf(w, "0x%08x, // c0x%04x (%s)%s %s\n", + c, i, s, wildcardStr(wildcard), nodeTypeStr(nodeType)) + } else { + fmt.Fprintf(w, "0x%x,\n", c) + } + } + fmt.Fprintf(w, "}\n\n") + fmt.Fprintf(w, "// max children %d (capacity %d)\n", maxChildren, 1<= 1<= 1<= 1< 0 && ss[0] == "" { + ss = ss[1:] + } + return ss +} + +// crush combines a list of strings, taking advantage of overlaps. It returns a +// single string that contains each input string as a substring. +func crush(ss []string) string { + maxLabelLen := 0 + for _, s := range ss { + if maxLabelLen < len(s) { + maxLabelLen = len(s) + } + } + + for prefixLen := maxLabelLen; prefixLen > 0; prefixLen-- { + prefixes := makePrefixMap(ss, prefixLen) + for i, s := range ss { + if len(s) <= prefixLen { + continue + } + mergeLabel(ss, i, prefixLen, prefixes) + } + } + + return strings.Join(ss, "") +} + +// mergeLabel merges the label at ss[i] with the first available matching label +// in prefixMap, where the last "prefixLen" characters in ss[i] match the first +// "prefixLen" characters in the matching label. +// It will merge ss[i] repeatedly until no more matches are available. +// All matching labels merged into ss[i] are replaced by "". +func mergeLabel(ss []string, i, prefixLen int, prefixes prefixMap) { + s := ss[i] + suffix := s[len(s)-prefixLen:] + for _, j := range prefixes[suffix] { + // Empty strings mean "already used." Also avoid merging with self. + if ss[j] == "" || i == j { + continue + } + if *v { + fmt.Fprintf(os.Stderr, "%d-length overlap at (%4d,%4d): %q and %q share %q\n", + prefixLen, i, j, ss[i], ss[j], suffix) + } + ss[i] += ss[j][prefixLen:] + ss[j] = "" + // ss[i] has a new suffix, so merge again if possible. + // Note: we only have to merge again at the same prefix length. Shorter + // prefix lengths will be handled in the next iteration of crush's for loop. + // Can there be matches for longer prefix lengths, introduced by the merge? + // I believe that any such matches would by necessity have been eliminated + // during substring removal or merged at a higher prefix length. For + // instance, in crush("abc", "cde", "bcdef"), combining "abc" and "cde" + // would yield "abcde", which could be merged with "bcdef." However, in + // practice "cde" would already have been elimintated by removeSubstrings. + mergeLabel(ss, i, prefixLen, prefixes) + return + } +} + +// prefixMap maps from a prefix to a list of strings containing that prefix. The +// list of strings is represented as indexes into a slice of strings stored +// elsewhere. +type prefixMap map[string][]int + +// makePrefixMap constructs a prefixMap from a slice of strings. +func makePrefixMap(ss []string, prefixLen int) prefixMap { + prefixes := make(prefixMap) + for i, s := range ss { + // We use < rather than <= because if a label matches on a prefix equal to + // its full length, that's actually a substring match handled by + // removeSubstrings. + if prefixLen < len(s) { + prefix := s[:prefixLen] + prefixes[prefix] = append(prefixes[prefix], i) + } + } + + return prefixes +} diff --git a/vendor/golang.org/x/net/publicsuffix/list.go b/vendor/golang.org/x/net/publicsuffix/list.go new file mode 100644 index 0000000..8bbf3bc --- /dev/null +++ b/vendor/golang.org/x/net/publicsuffix/list.go @@ -0,0 +1,135 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go run gen.go + +// Package publicsuffix provides a public suffix list based on data from +// http://publicsuffix.org/. A public suffix is one under which Internet users +// can directly register names. +package publicsuffix // import "golang.org/x/net/publicsuffix" + +// TODO: specify case sensitivity and leading/trailing dot behavior for +// func PublicSuffix and func EffectiveTLDPlusOne. + +import ( + "fmt" + "net/http/cookiejar" + "strings" +) + +// List implements the cookiejar.PublicSuffixList interface by calling the +// PublicSuffix function. +var List cookiejar.PublicSuffixList = list{} + +type list struct{} + +func (list) PublicSuffix(domain string) string { + ps, _ := PublicSuffix(domain) + return ps +} + +func (list) String() string { + return version +} + +// PublicSuffix returns the public suffix of the domain using a copy of the +// publicsuffix.org database compiled into the library. +// +// icann is whether the public suffix is managed by the Internet Corporation +// for Assigned Names and Numbers. If not, the public suffix is privately +// managed. For example, foo.org and foo.co.uk are ICANN domains, +// foo.dyndns.org and foo.blogspot.co.uk are private domains. +// +// Use cases for distinguishing ICANN domains like foo.com from private +// domains like foo.appspot.com can be found at +// https://wiki.mozilla.org/Public_Suffix_List/Use_Cases +func PublicSuffix(domain string) (publicSuffix string, icann bool) { + lo, hi := uint32(0), uint32(numTLD) + s, suffix, wildcard := domain, len(domain), false +loop: + for { + dot := strings.LastIndex(s, ".") + if wildcard { + suffix = 1 + dot + } + if lo == hi { + break + } + f := find(s[1+dot:], lo, hi) + if f == notFound { + break + } + + u := nodes[f] >> (nodesBitsTextOffset + nodesBitsTextLength) + icann = u&(1<>= nodesBitsICANN + u = children[u&(1<>= childrenBitsLo + hi = u & (1<>= childrenBitsHi + switch u & (1<>= childrenBitsNodeType + wildcard = u&(1<>= nodesBitsTextLength + offset := x & (1<