Skip to content

Commit

Permalink
同步到新 API 和一些修改
Browse files Browse the repository at this point in the history
  • Loading branch information
@SeaHOH
SeaHOH committed Aug 27, 2017
1 parent a9f648d commit a236ed0
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 128 deletions.
11 changes: 8 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
# GoProxy GAE Server
- 这算是增加冗余吧。相比原版作了以下修改：
- 上传程序，增加应用密码设置，省去手工修改。
- 上传程序，除标题 GoProxy 字样外其它描述都改为适应 GotoX。
- GAE 应用，修改首页版本检测时获取的链接到本仓库。
- 上传程序
- 增加应用密码设置，省去手工修改。
- 除标题 GoProxy 字样外其它描述都改为适应 GotoX。
- GAE 应用
- 支持不同的 debug 级别（0-2）。
- 不修改的代理请求的 `Accept-Encoding` 头域。
- 扩大检查 `Content-Encoding` 头域的匹配范围。
- 修改首页版本检测时获取的链接到本仓库。

# 使用
- Windows 用户：
Expand Down
246 changes: 123 additions & 123 deletions gae/gae.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,18 @@ import (
"bufio"
"bytes"
"compress/flate"
"compress/gzip"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"net/url"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strconv"
"strings"
"time"
Expand All @@ -23,30 +25,54 @@ import (
)

const (
Version = "1.0"
Version = "r9999"
Password = ""

DefaultFetchMaxSize = 1024 * 1024 * 4
DefaultDeadline = 20 * time.Second
DefaultOverquotaDelay = 4 * time.Second
DefaultURLFetchClosedDelay = 1 * time.Second
DefaultSSLVerify = false
DefaultDebug = 0
)

func IsBinary(b []byte) bool {
if len(b) > 32 {
b = b[:32]
}
if bytes.HasPrefix(b, []byte{0xef, 0xbb, 0xbf}) {
if len(b) > 3 && b[0] == 0xef && b[1] == 0xbb && b[2] == 0xbf {
// utf-8 text
return false
}
for _, c := range b {
if c == '\n' {
break
}
for i, c := range b {
if c > 0x7f {
return true
}
if c == '\n' && i > 4 {
break
}
if i > 32 {
break
}
}
return false
}

func IsTextContentType(contentType string) bool {
// text/* for html, plain text
// application/{json, javascript} for ajax
// application/{xml, rss, atom} for xml
// application/x-www-form-urlencoded for some video api
parts := strings.SplitN(contentType, "/", 2)
mct := parts[0]
if mct == "text" {
return true
}
if mct == "application" {
sct := strings.SplitN(parts[1], ";", 2)[0]
return sct == "json" ||
strings.HasSuffix(sct, "script") ||
strings.HasSuffix(sct, "xml") ||
strings.HasPrefix(sct, "xml") ||
strings.HasPrefix(sct, "rss") ||
strings.HasPrefix(sct, "atom") ||
sct == "x-www-form-urlencoded"
}
return false
}
Expand All @@ -65,9 +91,6 @@ func ReadRequest(r io.Reader) (req *http.Request, err error) {

req.Method = parts[0]
req.RequestURI = parts[1]
req.Proto = "HTTP/1.1"
req.ProtoMajor = 1
req.ProtoMinor = 1

if req.URL, err = url.Parse(req.RequestURI); err != nil {
return
Expand Down Expand Up @@ -109,12 +132,12 @@ func ReadRequest(r io.Reader) (req *http.Request, err error) {

func fmtError(c appengine.Context, err error) string {
return fmt.Sprintf(`{
"type": "appengine",
"type": "appengine(%s, %s/%s)",
"host": "%s",
"software": "%s",
"error": "%s"
}
`, appengine.DefaultVersionHostname(c), appengine.ServerSoftware(), err.Error())
`, runtime.Version(), runtime.GOOS, runtime.GOARCH, appengine.DefaultVersionHostname(c), appengine.ServerSoftware(), err.Error())
}

func handlerError(c appengine.Context, rw http.ResponseWriter, err error, code int) {
Expand All @@ -133,8 +156,8 @@ func handlerError(c appengine.Context, rw http.ResponseWriter, err error, code i
binary.BigEndian.PutUint16(b0, uint16(b.Len()))

rw.Header().Set("Content-Type", "image/gif")
rw.Header().Set("Content-Length", strconv.Itoa(len(b0)+b.Len()))
rw.WriteHeader(http.StatusOK)
//rw.Header().Set("Content-Length", strconv.Itoa(len(b0)+b.Len()))
//rw.WriteHeader(http.StatusOK)
rw.Write(b0)
rw.Write(b.Bytes())
}
Expand Down Expand Up @@ -162,81 +185,66 @@ func handler(rw http.ResponseWriter, r *http.Request) {
req.Body = r.Body
defer req.Body.Close()

params := http.Header{}
var paramPrefix string = http.CanonicalHeaderKey("X-UrlFetch-")
for key, values := range req.Header {
if strings.HasPrefix(key, paramPrefix) {
params.Set(key, values[0])
params := make(map[string]string)
if options := req.Header.Get("X-UrlFetch-Options"); options != "" {
for _, pair := range strings.Split(options, ",") {
parts := strings.SplitN(pair, "=", 2)
if len(parts) == 1 {
params[strings.TrimSpace(pair)] = ""
} else {
params[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
}
}
req.Header.Del("X-UrlFetch-Options")
}

for key, _ := range params {
req.Header.Del(key)
}

// req.Header.Del("X-Cloud-Trace-Context")
oAE := req.Header.Get("Accept-Encoding")
req.Header.Del("Accept-Encoding")

debugHeader := params.Get("X-UrlFetch-Debug")
debug := debugHeader != ""
debug := DefaultDebug
if s, ok := params["debug"]; ok && s != "" {
if n, err := strconv.Atoi(s); err == nil {
debug = n
}
}

if debug {
if debug > 1 {
c.Infof("Parsed Request=%#v\n", req)
}

if Password != "" {
password := params.Get("X-UrlFetch-Password")
switch {
case password == "":
password, ok := params["password"]
if !ok {
handlerError(c, rw, fmt.Errorf("urlfetch password required"), http.StatusForbidden)
return
case password != Password:
} else if password != Password {
handlerError(c, rw, fmt.Errorf("urlfetch password is wrong"), http.StatusForbidden)
return
}
}

deadline := DefaultDeadline
if s := params.Get("X-UrlFetch-Deadline"); s != "" {
if n, err := strconv.Atoi(s); err == nil {
deadline = time.Duration(n) * time.Second
}
}

overquotaDelay := DefaultOverquotaDelay
if s := params.Get("X-UrlFetch-OverquotaDelay"); s != "" {
if n, err := strconv.Atoi(s); err == nil {
overquotaDelay = time.Duration(n) * time.Second
}
}

urlfetchClosedDelay := DefaultURLFetchClosedDelay
if s := params.Get("X-UrlFetch-URLFetchClosedDelay"); s != "" {
if n, err := strconv.Atoi(s); err == nil {
urlfetchClosedDelay = time.Duration(n) * time.Second
}
}
//deadline := DefaultDeadline
//if s, ok := params["deadline"]; ok && s != "" {
// if n, err := strconv.Atoi(s); err == nil {
// deadline = time.Duration(n) * time.Second
// }
//}
deadline := 5

fetchMaxSize := DefaultFetchMaxSize
if s := params.Get("X-UrlFetch-MaxSize"); s != "" {
if s, ok := params["maxsize"]; ok && s != "" {
if n, err := strconv.Atoi(s); err == nil {
fetchMaxSize = n
}
}

sslVerify := DefaultSSLVerify
if s := params.Get("X-UrlFetch-SSLVerify"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
sslVerify = true
}
}
_, sslVerify := params["sslverify"]

var resp *http.Response
for i := 0; i < 2; i++ {
t := &urlfetch.Transport{
Context: c,
Deadline: deadline,
// useless now, set in Context
//Deadline: deadline,
AllowInvalidServerCertificate: !sslVerify,
}

Expand Down Expand Up @@ -280,10 +288,10 @@ func handler(rw http.ResponseWriter, r *http.Request) {
}
} else if strings.Contains(message, "Over quota") {
c.Warningf("URLFetchServiceError %T(%v) deadline=%v, url=%v", err, err, deadline, req.URL.String())
time.Sleep(overquotaDelay)
time.Sleep(DefaultOverquotaDelay)
} else if strings.Contains(message, "urlfetch: CLOSED") {
c.Warningf("URLFetchServiceError %T(%v) deadline=%v, url=%v", err, err, deadline, req.URL.String())
time.Sleep(urlfetchClosedDelay)
time.Sleep(DefaultURLFetchClosedDelay)
} else {
c.Errorf("URLFetchServiceError %T(%v) deadline=%v, url=%v", err, err, deadline, req.URL.String())
break
Expand All @@ -304,73 +312,65 @@ func handler(rw http.ResponseWriter, r *http.Request) {
resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
}

// urlfetch will decompress content, so try remove Content-Encoding
ce := resp.Header.Get("Content-Encoding")
ct := resp.Header.Get("Content-Type")
resp.Header.Del("Content-Encoding")

if (ce != "" ||
strings.HasPrefix(ct, "text/") ||
strings.HasPrefix(ct, "application/json") ||
strings.HasPrefix(ct, "application/x-javascript") ||
strings.HasPrefix(ct, "application/javascript") ||
strings.HasPrefix(ct, "application/x-www-form-urlencoded")) &&
resp.ContentLength > 1024 {
if v := reflect.ValueOf(resp.Body).Elem().FieldByName("content"); v.IsValid() {
var bb bytes.Buffer
var w io.WriteCloser
var ce1 string

switch {
case strings.Contains(oAE, "deflate"):
w, err = flate.NewWriter(&bb, flate.BestCompression)
ce1 = "deflate"
case strings.Contains(oAE, "gzip"):
w, err = gzip.NewWriterLevel(&bb, gzip.BestCompression)
ce1 = "gzip"
}

if err != nil {
handlerError(c, rw, err, http.StatusBadGateway)
return
}
oCE := resp.Header.Get("Content-Encoding")
chunked := false

if w != nil {
w.Write(v.Bytes())
w.Close()
// urlfetch will try to decompress the content when "Accept-Encoding" does not contain "gzip"
// delete "Content-Encoding" when content has decompressed with supported encoding
if !strings.Contains(oAE, "gzip") &&
(oCE == "gzip" || oCE == "deflate" || oCE == "br") {
resp.Header.Del("Content-Encoding")
oCE = ""
}

bbLen := int64(bb.Len())
if bbLen < resp.ContentLength {
resp.Body = ioutil.NopCloser(&bb)
resp.ContentLength = bbLen
resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
resp.Header.Set("Content-Encoding", ce1)
}
if oCE == "" &&
IsTextContentType(resp.Header.Get("Content-Type")) {
content := reflect.ValueOf(resp.Body).Elem().FieldByName("content").Bytes()
if IsBinary(content) {
// urlfetch will remove "Content-Encoding: deflate" when "Accept-Encoding" contains "gzip"
ext := filepath.Ext(req.URL.Path)
if ext == "" || IsTextContentType(mime.TypeByExtension(ext)) {
// ignore wrong "Content-Type"
resp.Header.Set("Content-Encoding", "deflate")
}
} else {
chunked = true
}
}

if debug {
if debug > 1 {
c.Infof("Write Response=%#v\n", resp)
}

c.Infof("%s \"%s %s %s\" %d %s", resp.Request.RemoteAddr, resp.Request.Method, resp.Request.URL.String(), resp.Request.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
if debug > 0 {
c.Infof("%s \"%s %s %s\" %d %s", resp.Request.RemoteAddr, resp.Request.Method, resp.Request.URL.String(), resp.Request.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
}

var b bytes.Buffer
w, _ := flate.NewWriter(&b, flate.BestCompression)
fmt.Fprintf(w, "HTTP/1.1 %s\r\n", resp.Status)
resp.Header.Write(w)
io.WriteString(w, "\r\n")
w.Close()
b := &bytes.Buffer{}

b0 := []byte{0, 0}
binary.BigEndian.PutUint16(b0, uint16(b.Len()))
if chunked {
fmt.Fprintf(b, "HTTP/1.1 %s\r\n", resp.Status)
resp.Header.Write(b)
io.WriteString(b, "\r\n")

rw.Header().Set("Content-Type", "image/gif")
rw.Header().Set("Content-Length", strconv.FormatInt(int64(len(b0)+b.Len())+resp.ContentLength, 10))
rw.WriteHeader(http.StatusOK)
rw.Write(b0)
io.Copy(rw, io.MultiReader(&b, resp.Body))
rw.Header().Set("Content-Type", "text/plain")
} else {
w, _ := flate.NewWriter(b, flate.BestCompression)
fmt.Fprintf(w, "HTTP/1.1 %s\r\n", resp.Status)
resp.Header.Write(w)
io.WriteString(w, "\r\n")
w.Close()

b0 := []byte{0, 0}
binary.BigEndian.PutUint16(b0, uint16(b.Len()))

rw.Header().Set("Content-Type", "image/gif")
// we need not set a "Content-Length" header, App Engine will reset it
//rw.Header().Set("Content-Length", strconv.FormatInt(int64(len(b0)+b.Len())+resp.ContentLength, 10))
//rw.WriteHeader(http.StatusOK)
rw.Write(b0)
}
io.Copy(rw, io.MultiReader(b, resp.Body))
}

func favicon(rw http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -414,12 +414,12 @@ func root(rw http.ResponseWriter, r *http.Request) {
message = "please update this server"
}
fmt.Fprintf(rw, `{
"server": "goproxy %s"
"server": "goproxy %s (%s, %s/%s)"
"latest": "%s",
"deploy": "%s",
"message": "%s"
}
`, Version, latest, ctime, message)
`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, latest, ctime, message)
}

func init() {
Expand Down
Loading

0 comments on commit a236ed0

Please sign in to comment.