From 97a373f5e7cda54fa490a3a6993a0b839ff18b01 Mon Sep 17 00:00:00 2001
From: Unknwon <joe2010xtmf@163.com>
Date: Sat, 23 Aug 2014 21:13:55 +0800
Subject: [PATCH] Fix #362, update code with upstream
---
modules/base/markdown.go | 2 +-
modules/httplib/httplib.go | 296 ++++++++++++++++++++++----------
modules/httplib/httplib_test.go | 196 +++++++++++++++++++--
3 files changed, 387 insertions(+), 107 deletions(-)
mode change 100755 => 100644 modules/httplib/httplib.go
mode change 100755 => 100644 modules/httplib/httplib_test.go
diff --git a/modules/base/markdown.go b/modules/base/markdown.go
index 0825decb4..b8bd33e11 100644
--- a/modules/base/markdown.go
+++ b/modules/base/markdown.go
@@ -39,7 +39,7 @@ func isLink(link []byte) bool {
func IsMarkdownFile(name string) bool {
name = strings.ToLower(name)
switch filepath.Ext(name) {
- case ".md", ".markdown", ".mdown":
+ case ".md", ".markdown", ".mdown", ".mkd":
return true
}
return false
diff --git a/modules/httplib/httplib.go b/modules/httplib/httplib.go
old mode 100755
new mode 100644
index 35870c1f1..5f592168d
--- a/modules/httplib/httplib.go
+++ b/modules/httplib/httplib.go
@@ -5,6 +5,8 @@
package httplib
+// NOTE: last sync c07b1d8 on Aug 23, 2014.
+
import (
"bytes"
"crypto/tls"
@@ -12,91 +14,143 @@ import (
"encoding/xml"
"io"
"io/ioutil"
+ "mime/multipart"
"net"
"net/http"
+ "net/http/cookiejar"
"net/http/httputil"
"net/url"
"os"
"strings"
+ "sync"
"time"
)
-var defaultUserAgent = "gogsServer"
+var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false}
+var defaultCookieJar http.CookieJar
+var settingMutex sync.Mutex
+
+// createDefaultCookie creates a global cookiejar to store cookies.
+func createDefaultCookie() {
+ settingMutex.Lock()
+ defer settingMutex.Unlock()
+ defaultCookieJar, _ = cookiejar.New(nil)
+}
+
+// Overwrite default settings
+func SetDefaultSetting(setting BeegoHttpSettings) {
+ settingMutex.Lock()
+ defer settingMutex.Unlock()
+ defaultSetting = setting
+ if defaultSetting.ConnectTimeout == 0 {
+ defaultSetting.ConnectTimeout = 60 * time.Second
+ }
+ if defaultSetting.ReadWriteTimeout == 0 {
+ defaultSetting.ReadWriteTimeout = 60 * time.Second
+ }
+}
+
+// return *BeegoHttpRequest with specific method
+func newBeegoRequest(url, method string) *BeegoHttpRequest {
+ var resp http.Response
+ req := http.Request{
+ Method: method,
+ Header: make(http.Header),
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ }
+ return &BeegoHttpRequest{url, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil}
+}
// Get returns *BeegoHttpRequest with GET method.
func Get(url string) *BeegoHttpRequest {
- var req http.Request
- req.Method = "GET"
- req.Header = http.Header{}
- req.Header.Set("User-Agent", defaultUserAgent)
- return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+ return newBeegoRequest(url, "GET")
}
// Post returns *BeegoHttpRequest with POST method.
func Post(url string) *BeegoHttpRequest {
- var req http.Request
- req.Method = "POST"
- req.Header = http.Header{}
- req.Header.Set("User-Agent", defaultUserAgent)
- return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+ return newBeegoRequest(url, "POST")
}
// Put returns *BeegoHttpRequest with PUT method.
func Put(url string) *BeegoHttpRequest {
- var req http.Request
- req.Method = "PUT"
- req.Header = http.Header{}
- req.Header.Set("User-Agent", defaultUserAgent)
- return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+ return newBeegoRequest(url, "PUT")
}
-// Delete returns *BeegoHttpRequest DELETE GET method.
+// Delete returns *BeegoHttpRequest DELETE method.
func Delete(url string) *BeegoHttpRequest {
- var req http.Request
- req.Method = "DELETE"
- req.Header = http.Header{}
- req.Header.Set("User-Agent", defaultUserAgent)
- return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+ return newBeegoRequest(url, "DELETE")
}
// Head returns *BeegoHttpRequest with HEAD method.
func Head(url string) *BeegoHttpRequest {
- var req http.Request
- req.Method = "HEAD"
- req.Header = http.Header{}
- req.Header.Set("User-Agent", defaultUserAgent)
- return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+ return newBeegoRequest(url, "HEAD")
+}
+
+// BeegoHttpSettings
+type BeegoHttpSettings struct {
+ ShowDebug bool
+ UserAgent string
+ ConnectTimeout time.Duration
+ ReadWriteTimeout time.Duration
+ TlsClientConfig *tls.Config
+ Proxy func(*http.Request) (*url.URL, error)
+ Transport http.RoundTripper
+ EnableCookie bool
}
// BeegoHttpRequest provides more useful methods for requesting one url than http.Request.
type BeegoHttpRequest struct {
- url string
- req *http.Request
- params map[string]string
- showdebug bool
- connectTimeout time.Duration
- readWriteTimeout time.Duration
- tlsClientConfig *tls.Config
- proxy func(*http.Request) (*url.URL, error)
- transport http.RoundTripper
+ url string
+ req *http.Request
+ params map[string]string
+ files map[string]string
+ setting BeegoHttpSettings
+ resp *http.Response
+ body []byte
+}
+
+// Change request settings
+func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest {
+ b.setting = setting
+ return b
+}
+
+// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
+func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest {
+ b.req.SetBasicAuth(username, password)
+ return b
+}
+
+// SetEnableCookie sets enable/disable cookiejar
+func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest {
+ b.setting.EnableCookie = enable
+ return b
+}
+
+// SetUserAgent sets User-Agent header field
+func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest {
+ b.setting.UserAgent = useragent
+ return b
}
// Debug sets show debug or not when executing request.
func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest {
- b.showdebug = isdebug
+ b.setting.ShowDebug = isdebug
return b
}
// SetTimeout sets connect time out and read-write time out for BeegoRequest.
func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest {
- b.connectTimeout = connectTimeout
- b.readWriteTimeout = readWriteTimeout
+ b.setting.ConnectTimeout = connectTimeout
+ b.setting.ReadWriteTimeout = readWriteTimeout
return b
}
// SetTLSClientConfig sets tls connection configurations if visiting https url.
func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest {
- b.tlsClientConfig = config
+ b.setting.TlsClientConfig = config
return b
}
@@ -106,6 +160,23 @@ func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest {
return b
}
+// Set the protocol version for incoming requests.
+// Client requests always use HTTP/1.1.
+func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest {
+ if len(vers) == 0 {
+ vers = "HTTP/1.1"
+ }
+
+ major, minor, ok := http.ParseHTTPVersion(vers)
+ if ok {
+ b.req.Proto = vers
+ b.req.ProtoMajor = major
+ b.req.ProtoMinor = minor
+ }
+
+ return b
+}
+
// SetCookie add cookie into request.
func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
b.req.Header.Add("Cookie", cookie.String())
@@ -114,7 +185,7 @@ func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
// Set transport to
func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest {
- b.transport = transport
+ b.setting.Transport = transport
return b
}
@@ -126,7 +197,7 @@ func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpR
// return u, nil
// }
func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest {
- b.proxy = proxy
+ b.setting.Proxy = proxy
return b
}
@@ -137,6 +208,11 @@ func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest {
return b
}
+func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest {
+ b.files[formname] = filename
+ return b
+}
+
// Body adds request raw body.
// it supports string and []byte.
func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
@@ -154,6 +230,9 @@ func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
}
func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
+ if b.resp.StatusCode != 0 {
+ return b.resp, nil
+ }
var paramBody string
if len(b.params) > 0 {
var buf bytes.Buffer
@@ -174,21 +253,90 @@ func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
b.url = b.url + "?" + paramBody
}
} else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 {
- b.Header("Content-Type", "application/x-www-form-urlencoded")
- b.Body(paramBody)
+ if len(b.files) > 0 {
+ bodyBuf := &bytes.Buffer{}
+ bodyWriter := multipart.NewWriter(bodyBuf)
+ for formname, filename := range b.files {
+ fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
+ if err != nil {
+ return nil, err
+ }
+ fh, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ //iocopy
+ _, err = io.Copy(fileWriter, fh)
+ fh.Close()
+ if err != nil {
+ return nil, err
+ }
+ }
+ for k, v := range b.params {
+ bodyWriter.WriteField(k, v)
+ }
+ contentType := bodyWriter.FormDataContentType()
+ bodyWriter.Close()
+ b.Header("Content-Type", contentType)
+ b.req.Body = ioutil.NopCloser(bodyBuf)
+ b.req.ContentLength = int64(bodyBuf.Len())
+ } else {
+ b.Header("Content-Type", "application/x-www-form-urlencoded")
+ b.Body(paramBody)
+ }
}
url, err := url.Parse(b.url)
- if url.Scheme == "" {
- b.url = "http://" + b.url
- url, err = url.Parse(b.url)
- }
if err != nil {
return nil, err
}
b.req.URL = url
- if b.showdebug {
+
+ trans := b.setting.Transport
+
+ if trans == nil {
+ // create default transport
+ trans = &http.Transport{
+ TLSClientConfig: b.setting.TlsClientConfig,
+ Proxy: b.setting.Proxy,
+ Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
+ }
+ } else {
+ // if b.transport is *http.Transport then set the settings.
+ if t, ok := trans.(*http.Transport); ok {
+ if t.TLSClientConfig == nil {
+ t.TLSClientConfig = b.setting.TlsClientConfig
+ }
+ if t.Proxy == nil {
+ t.Proxy = b.setting.Proxy
+ }
+ if t.Dial == nil {
+ t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
+ }
+ }
+ }
+
+ var jar http.CookieJar
+ if b.setting.EnableCookie {
+ if defaultCookieJar == nil {
+ createDefaultCookie()
+ }
+ jar = defaultCookieJar
+ } else {
+ jar = nil
+ }
+
+ client := &http.Client{
+ Transport: trans,
+ Jar: jar,
+ }
+
+ if b.setting.UserAgent != "" {
+ b.req.Header.Set("User-Agent", b.setting.UserAgent)
+ }
+
+ if b.setting.ShowDebug {
dump, err := httputil.DumpRequest(b.req, true)
if err != nil {
println(err.Error())
@@ -196,38 +344,11 @@ func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
println(string(dump))
}
- trans := b.transport
-
- if trans == nil {
- // create default transport
- trans = &http.Transport{
- TLSClientConfig: b.tlsClientConfig,
- Proxy: b.proxy,
- Dial: TimeoutDialer(b.connectTimeout, b.readWriteTimeout),
- }
- } else {
- // if b.transport is *http.Transport then set the settings.
- if t, ok := trans.(*http.Transport); ok {
- if t.TLSClientConfig == nil {
- t.TLSClientConfig = b.tlsClientConfig
- }
- if t.Proxy == nil {
- t.Proxy = b.proxy
- }
- if t.Dial == nil {
- t.Dial = TimeoutDialer(b.connectTimeout, b.readWriteTimeout)
- }
- }
- }
-
- client := &http.Client{
- Transport: trans,
- }
-
resp, err := client.Do(b.req)
if err != nil {
return nil, err
}
+ b.resp = resp
return resp, nil
}
@@ -245,6 +366,9 @@ func (b *BeegoHttpRequest) String() (string, error) {
// Bytes returns the body []byte in response.
// it calls Response inner.
func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
+ if b.body != nil {
+ return b.body, nil
+ }
resp, err := b.getResponse()
if err != nil {
return nil, err
@@ -257,6 +381,7 @@ func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
if err != nil {
return nil, err
}
+ b.body = data
return data, nil
}
@@ -278,10 +403,7 @@ func (b *BeegoHttpRequest) ToFile(filename string) error {
}
defer resp.Body.Close()
_, err = io.Copy(f, resp.Body)
- if err != nil {
- return err
- }
- return nil
+ return err
}
// ToJson returns the map that marshals from the body bytes as json in response .
@@ -292,24 +414,18 @@ func (b *BeegoHttpRequest) ToJson(v interface{}) error {
return err
}
err = json.Unmarshal(data, v)
- if err != nil {
- return err
- }
- return nil
+ return err
}
// ToXml returns the map that marshals from the body bytes as xml in response .
// it calls Response inner.
-func (b *BeegoHttpRequest) ToXML(v interface{}) error {
+func (b *BeegoHttpRequest) ToXml(v interface{}) error {
data, err := b.Bytes()
if err != nil {
return err
}
err = xml.Unmarshal(data, v)
- if err != nil {
- return err
- }
- return nil
+ return err
}
// Response executes request client gets response mannually.
diff --git a/modules/httplib/httplib_test.go b/modules/httplib/httplib_test.go
old mode 100755
new mode 100644
index cc56dd881..8a529c53a
--- a/modules/httplib/httplib_test.go
+++ b/modules/httplib/httplib_test.go
@@ -1,32 +1,196 @@
+// Copyright 2013 The Beego Authors. All rights reserved.
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
package httplib
import (
"io/ioutil"
+ "os"
+ "strings"
"testing"
)
-func TestGetUrl(t *testing.T) {
- resp, err := Get("http://beego.me/").Debug(true).Response()
+func TestResponse(t *testing.T) {
+ req := Get("http://httpbin.org/get")
+ resp, err := req.Response()
if err != nil {
t.Fatal(err)
}
- if resp.Body == nil {
- t.Fatal("body is nil")
- }
- data, err := ioutil.ReadAll(resp.Body)
- defer resp.Body.Close()
- if err != nil {
- t.Fatal(err)
- }
- if len(data) == 0 {
- t.Fatal("data is no")
- }
+ t.Log(resp)
+}
- str, err := Get("http://beego.me/").String()
+func TestGet(t *testing.T) {
+ req := Get("http://httpbin.org/get")
+ b, err := req.Bytes()
if err != nil {
t.Fatal(err)
}
- if len(str) == 0 {
- t.Fatal("has no info")
+ t.Log(b)
+
+ s, err := req.String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(s)
+
+ if string(b) != s {
+ t.Fatal("request data not match")
+ }
+}
+
+func TestSimplePost(t *testing.T) {
+ v := "smallfish"
+ req := Post("http://httpbin.org/post")
+ req.Param("username", v)
+
+ str, err := req.String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ n := strings.Index(str, v)
+ if n == -1 {
+ t.Fatal(v + " not found in post")
+ }
+}
+
+func TestPostFile(t *testing.T) {
+ v := "smallfish"
+ req := Post("http://httpbin.org/post")
+ req.Param("username", v)
+ req.PostFile("uploadfile", "httplib_test.go")
+
+ str, err := req.String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ n := strings.Index(str, v)
+ if n == -1 {
+ t.Fatal(v + " not found in post")
+ }
+}
+
+func TestSimplePut(t *testing.T) {
+ str, err := Put("http://httpbin.org/put").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+}
+
+func TestSimpleDelete(t *testing.T) {
+ str, err := Delete("http://httpbin.org/delete").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+}
+
+func TestWithCookie(t *testing.T) {
+ v := "smallfish"
+ str, err := Get("http://httpbin.org/cookies/set?k1=" + v).SetEnableCookie(true).String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ str, err = Get("http://httpbin.org/cookies").SetEnableCookie(true).String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ n := strings.Index(str, v)
+ if n == -1 {
+ t.Fatal(v + " not found in cookie")
+ }
+}
+
+func TestWithBasicAuth(t *testing.T) {
+ str, err := Get("http://httpbin.org/basic-auth/user/passwd").SetBasicAuth("user", "passwd").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+ n := strings.Index(str, "authenticated")
+ if n == -1 {
+ t.Fatal("authenticated not found in response")
+ }
+}
+
+func TestWithUserAgent(t *testing.T) {
+ v := "beego"
+ str, err := Get("http://httpbin.org/headers").SetUserAgent(v).String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ n := strings.Index(str, v)
+ if n == -1 {
+ t.Fatal(v + " not found in user-agent")
+ }
+}
+
+func TestWithSetting(t *testing.T) {
+ v := "beego"
+ var setting BeegoHttpSettings
+ setting.EnableCookie = true
+ setting.UserAgent = v
+ setting.Transport = nil
+ SetDefaultSetting(setting)
+
+ str, err := Get("http://httpbin.org/get").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(str)
+
+ n := strings.Index(str, v)
+ if n == -1 {
+ t.Fatal(v + " not found in user-agent")
+ }
+}
+
+func TestToJson(t *testing.T) {
+ req := Get("http://httpbin.org/ip")
+ resp, err := req.Response()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(resp)
+
+ // httpbin will return http remote addr
+ type Ip struct {
+ Origin string `json:"origin"`
+ }
+ var ip Ip
+ err = req.ToJson(&ip)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(ip.Origin)
+
+ if n := strings.Count(ip.Origin, "."); n != 3 {
+ t.Fatal("response is not valid ip")
+ }
+}
+
+func TestToFile(t *testing.T) {
+ f := "beego_testfile"
+ req := Get("http://httpbin.org/ip")
+ err := req.ToFile(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.Remove(f)
+ b, err := ioutil.ReadFile(f)
+ if n := strings.Index(string(b), "origin"); n == -1 {
+ t.Fatal(err)
}
}