added logging, cleaned up

main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"github.com/foomo/ContentServer/server"
+	"github.com/foomo/ContentServer/server/log"
 )
 
 func main() {
-	server.Run(":8080", "http://test.bestbytes/foomo/modules/Foomo.Page.Content/services/content.php")
+	log.SetLogLevel(log.LOG_LEVEL_DEBUG)
+	//server.Run(":8080", "http://test.bestbytes/foomo/modules/Foomo.Page.Content/services/content.php")
+	server.RunSocketServer("http://test.bestbytes/foomo/modules/Foomo.Page.Content/services/content.php", "0.0.0.0:8081")
 }

server/http.go

@@ -0,0 +1,71 @@
+package server
+
+import (
+	"fmt"
+	"github.com/foomo/ContentServer/server/log"
+	"github.com/foomo/ContentServer/server/repo"
+	"github.com/foomo/ContentServer/server/requests"
+	"github.com/foomo/ContentServer/server/utils"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+func contentHandler(w http.ResponseWriter, r *http.Request) {
+	request := requests.NewContent()
+	utils.PopulateRequest(r, request)
+	utils.JsonResponse(w, contentRepo.GetContent(request))
+}
+
+func uriHandler(w http.ResponseWriter, r *http.Request) {
+	parts := strings.Split(r.RequestURI, "/")
+	if len(parts) == 5 {
+		utils.JsonResponse(w, contentRepo.GetURI(parts[2], parts[3], parts[4]))
+	} else {
+		wtfHandler(w, r)
+	}
+}
+
+func wtfHandler(w http.ResponseWriter, r *http.Request) {
+	msg := "unhandled request: " + r.RequestURI
+	fmt.Fprint(w, msg, "\n")
+	log.Error(msg)
+}
+
+func update() interface{} {
+	contentRepo.Update()
+	return contentRepo.Directory
+}
+
+func commandHandler(w http.ResponseWriter, r *http.Request) {
+	parts := strings.Split(r.RequestURI, "/")
+	switch true {
+	case true == (len(parts) > 1):
+		switch parts[2] {
+		case "update":
+			utils.JsonResponse(w, update())
+			return
+		default:
+			help := make(map[string]interface{})
+			help["input"] = parts[2]
+			help["commands"] = []string{"update", "help", "content"}
+			utils.JsonResponse(w, help)
+		}
+	default:
+		wtfHandler(w, r)
+	}
+}
+
+func Run(addr string, serverUrl string) {
+	log.Notice("staring content server")
+	log.Notice("  loading content from " + serverUrl)
+	contentRepo = repo.NewRepo(serverUrl)
+	contentRepo.Update()
+	log.Notice("    loaded " + strconv.Itoa(len(contentRepo.Directory)) + " items")
+	http.HandleFunc("/content", contentHandler)
+	http.HandleFunc("/uri/", uriHandler)
+	http.HandleFunc("/cmd/", commandHandler)
+	http.HandleFunc("/", wtfHandler)
+	log.Notice("  starting service on " + addr)
+	http.ListenAndServe(addr, nil)
+}

server/jjson/bench_test.go

@@ -0,0 +1,189 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Large data benchmark.
+// The JSON data is a summary of agl's changes in the
+// go, webkit, and chromium open source projects.
+// We benchmark converting between the JSON form
+// and in-memory data structures.
+
+package jjson
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+type codeResponse struct {
+	Tree     *codeNode `json:"tree"`
+	Username string    `json:"username"`
+}
+
+type codeNode struct {
+	Name     string      `json:"name"`
+	Kids     []*codeNode `json:"kids"`
+	CLWeight float64     `json:"cl_weight"`
+	Touches  int         `json:"touches"`
+	MinT     int64       `json:"min_t"`
+	MaxT     int64       `json:"max_t"`
+	MeanT    int64       `json:"mean_t"`
+}
+
+var codeJSON []byte
+var codeStruct codeResponse
+
+func codeInit() {
+	f, err := os.Open("testdata/code.json.gz")
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(gz)
+	if err != nil {
+		panic(err)
+	}
+
+	codeJSON = data
+
+	if err := Unmarshal(codeJSON, &codeStruct); err != nil {
+		panic("unmarshal code.json: " + err.Error())
+	}
+
+	if data, err = Marshal(&codeStruct); err != nil {
+		panic("marshal code.json: " + err.Error())
+	}
+
+	if !bytes.Equal(data, codeJSON) {
+		println("different lengths", len(data), len(codeJSON))
+		for i := 0; i < len(data) && i < len(codeJSON); i++ {
+			if data[i] != codeJSON[i] {
+				println("re-marshal: changed at byte", i)
+				println("orig: ", string(codeJSON[i-10:i+10]))
+				println("new: ", string(data[i-10:i+10]))
+				break
+			}
+		}
+		panic("re-marshal code.json: different result")
+	}
+}
+
+func BenchmarkCodeEncoder(b *testing.B) {
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	enc := NewEncoder(ioutil.Discard)
+	for i := 0; i < b.N; i++ {
+		if err := enc.Encode(&codeStruct); err != nil {
+			b.Fatal("Encode:", err)
+		}
+	}
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeMarshal(b *testing.B) {
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	for i := 0; i < b.N; i++ {
+		if _, err := Marshal(&codeStruct); err != nil {
+			b.Fatal("Marshal:", err)
+		}
+	}
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeDecoder(b *testing.B) {
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	var buf bytes.Buffer
+	dec := NewDecoder(&buf)
+	var r codeResponse
+	for i := 0; i < b.N; i++ {
+		buf.Write(codeJSON)
+		// hide EOF
+		buf.WriteByte('\n')
+		buf.WriteByte('\n')
+		buf.WriteByte('\n')
+		if err := dec.Decode(&r); err != nil {
+			b.Fatal("Decode:", err)
+		}
+	}
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeUnmarshal(b *testing.B) {
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	for i := 0; i < b.N; i++ {
+		var r codeResponse
+		if err := Unmarshal(codeJSON, &r); err != nil {
+			b.Fatal("Unmmarshal:", err)
+		}
+	}
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeUnmarshalReuse(b *testing.B) {
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	var r codeResponse
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(codeJSON, &r); err != nil {
+			b.Fatal("Unmmarshal:", err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalString(b *testing.B) {
+	data := []byte(`"hello, world"`)
+	var s string
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(data, &s); err != nil {
+			b.Fatal("Unmarshal:", err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalFloat64(b *testing.B) {
+	var f float64
+	data := []byte(`3.14`)
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(data, &f); err != nil {
+			b.Fatal("Unmarshal:", err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalInt64(b *testing.B) {
+	var x int64
+	data := []byte(`3`)
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(data, &x); err != nil {
+			b.Fatal("Unmarshal:", err)
+		}
+	}
+}

server/jjson/encode_test.go

@@ -0,0 +1,403 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"testing"
+	"unicode"
+)
+
+type Optionals struct {
+	Sr string `json:"sr"`
+	So string `json:"so,omitempty"`
+	Sw string `json:"-"`
+
+	Ir int `json:"omitempty"` // actually named omitempty, not an option
+	Io int `json:"io,omitempty"`
+
+	Slr []string `json:"slr,random"`
+	Slo []string `json:"slo,omitempty"`
+
+	Mr map[string]interface{} `json:"mr"`
+	Mo map[string]interface{} `json:",omitempty"`
+}
+
+var optionalsExpected = `{
+ "sr": "",
+ "omitempty": 0,
+ "slr": null,
+ "mr": {}
+}`
+
+func TestOmitEmpty(t *testing.T) {
+	var o Optionals
+	o.Sw = "something"
+	o.Mr = map[string]interface{}{}
+	o.Mo = map[string]interface{}{}
+
+	got, err := MarshalIndent(&o, "", " ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(got); got != optionalsExpected {
+		t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected)
+	}
+}
+
+type StringTag struct {
+	BoolStr bool   `json:",string"`
+	IntStr  int64  `json:",string"`
+	StrStr  string `json:",string"`
+}
+
+var stringTagExpected = `{
+ "BoolStr": "true",
+ "IntStr": "42",
+ "StrStr": "\"xzbit\""
+}`
+
+func TestStringTag(t *testing.T) {
+	var s StringTag
+	s.BoolStr = true
+	s.IntStr = 42
+	s.StrStr = "xzbit"
+	got, err := MarshalIndent(&s, "", " ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(got); got != stringTagExpected {
+		t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected)
+	}
+
+	// Verify that it round-trips.
+	var s2 StringTag
+	err = NewDecoder(bytes.NewBuffer(got)).Decode(&s2)
+	if err != nil {
+		t.Fatalf("Decode: %v", err)
+	}
+	if !reflect.DeepEqual(s, s2) {
+		t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2)
+	}
+}
+
+// byte slices are special even if they're renamed types.
+type renamedByte byte
+type renamedByteSlice []byte
+type renamedRenamedByteSlice []renamedByte
+
+func TestEncodeRenamedByteSlice(t *testing.T) {
+	s := renamedByteSlice("abc")
+	result, err := Marshal(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expect := `"YWJj"`
+	if string(result) != expect {
+		t.Errorf(" got %s want %s", result, expect)
+	}
+	r := renamedRenamedByteSlice("abc")
+	result, err = Marshal(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(result) != expect {
+		t.Errorf(" got %s want %s", result, expect)
+	}
+}
+
+var unsupportedValues = []interface{}{
+	math.NaN(),
+	math.Inf(-1),
+	math.Inf(1),
+}
+
+func TestUnsupportedValues(t *testing.T) {
+	for _, v := range unsupportedValues {
+		if _, err := Marshal(v); err != nil {
+			if _, ok := err.(*UnsupportedValueError); !ok {
+				t.Errorf("for %v, got %T want UnsupportedValueError", v, err)
+			}
+		} else {
+			t.Errorf("for %v, expected error", v)
+		}
+	}
+}
+
+// Ref has Marshaler and Unmarshaler methods with pointer receiver.
+type Ref int
+
+func (*Ref) MarshalJSON() ([]byte, error) {
+	return []byte(`"ref"`), nil
+}
+
+func (r *Ref) UnmarshalJSON([]byte) error {
+	*r = 12
+	return nil
+}
+
+// Val has Marshaler methods with value receiver.
+type Val int
+
+func (Val) MarshalJSON() ([]byte, error) {
+	return []byte(`"val"`), nil
+}
+
+// RefText has Marshaler and Unmarshaler methods with pointer receiver.
+type RefText int
+
+func (*RefText) MarshalText() ([]byte, error) {
+	return []byte(`"ref"`), nil
+}
+
+func (r *RefText) UnmarshalText([]byte) error {
+	*r = 13
+	return nil
+}
+
+// ValText has Marshaler methods with value receiver.
+type ValText int
+
+func (ValText) MarshalText() ([]byte, error) {
+	return []byte(`"val"`), nil
+}
+
+func TestRefValMarshal(t *testing.T) {
+	var s = struct {
+		R0 Ref
+		R1 *Ref
+		R2 RefText
+		R3 *RefText
+		V0 Val
+		V1 *Val
+		V2 ValText
+		V3 *ValText
+	}{
+		R0: 12,
+		R1: new(Ref),
+		R2: 14,
+		R3: new(RefText),
+		V0: 13,
+		V1: new(Val),
+		V2: 15,
+		V3: new(ValText),
+	}
+	const want = `{"R0":"ref","R1":"ref","R2":"\"ref\"","R3":"\"ref\"","V0":"val","V1":"val","V2":"\"val\"","V3":"\"val\""}`
+	b, err := Marshal(&s)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if got := string(b); got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}
+
+// C implements Marshaler and returns unescaped JSON.
+type C int
+
+func (C) MarshalJSON() ([]byte, error) {
+	return []byte(`"<&>"`), nil
+}
+
+// CText implements Marshaler and returns unescaped text.
+type CText int
+
+func (CText) MarshalText() ([]byte, error) {
+	return []byte(`"<&>"`), nil
+}
+
+func TestMarshalerEscaping(t *testing.T) {
+	var c C
+	want := `"\u003c\u0026\u003e"`
+	b, err := Marshal(c)
+	if err != nil {
+		t.Fatalf("Marshal(c): %v", err)
+	}
+	if got := string(b); got != want {
+		t.Errorf("Marshal(c) = %#q, want %#q", got, want)
+	}
+
+	var ct CText
+	want = `"\"\u003c\u0026\u003e\""`
+	b, err = Marshal(ct)
+	if err != nil {
+		t.Fatalf("Marshal(ct): %v", err)
+	}
+	if got := string(b); got != want {
+		t.Errorf("Marshal(ct) = %#q, want %#q", got, want)
+	}
+}
+
+type IntType int
+
+type MyStruct struct {
+	IntType
+}
+
+func TestAnonymousNonstruct(t *testing.T) {
+	var i IntType = 11
+	a := MyStruct{i}
+	const want = `{"IntType":11}`
+
+	b, err := Marshal(a)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if got := string(b); got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}
+
+type BugA struct {
+	S string
+}
+
+type BugB struct {
+	BugA
+	S string
+}
+
+type BugC struct {
+	S string
+}
+
+// Legal Go: We never use the repeated embedded field (S).
+type BugX struct {
+	A int
+	BugA
+	BugB
+}
+
+// Issue 5245.
+func TestEmbeddedBug(t *testing.T) {
+	v := BugB{
+		BugA{"A"},
+		"B",
+	}
+	b, err := Marshal(v)
+	if err != nil {
+		t.Fatal("Marshal:", err)
+	}
+	want := `{"S":"B"}`
+	got := string(b)
+	if got != want {
+		t.Fatalf("Marshal: got %s want %s", got, want)
+	}
+	// Now check that the duplicate field, S, does not appear.
+	x := BugX{
+		A: 23,
+	}
+	b, err = Marshal(x)
+	if err != nil {
+		t.Fatal("Marshal:", err)
+	}
+	want = `{"A":23}`
+	got = string(b)
+	if got != want {
+		t.Fatalf("Marshal: got %s want %s", got, want)
+	}
+}
+
+type BugD struct { // Same as BugA after tagging.
+	XXX string `json:"S"`
+}
+
+// BugD's tagged S field should dominate BugA's.
+type BugY struct {
+	BugA
+	BugD
+}
+
+// Test that a field with a tag dominates untagged fields.
+func TestTaggedFieldDominates(t *testing.T) {
+	v := BugY{
+		BugA{"BugA"},
+		BugD{"BugD"},
+	}
+	b, err := Marshal(v)
+	if err != nil {
+		t.Fatal("Marshal:", err)
+	}
+	want := `{"S":"BugD"}`
+	got := string(b)
+	if got != want {
+		t.Fatalf("Marshal: got %s want %s", got, want)
+	}
+}
+
+// There are no tags here, so S should not appear.
+type BugZ struct {
+	BugA
+	BugC
+	BugY // Contains a tagged S field through BugD; should not dominate.
+}
+
+func TestDuplicatedFieldDisappears(t *testing.T) {
+	v := BugZ{
+		BugA{"BugA"},
+		BugC{"BugC"},
+		BugY{
+			BugA{"nested BugA"},
+			BugD{"nested BugD"},
+		},
+	}
+	b, err := Marshal(v)
+	if err != nil {
+		t.Fatal("Marshal:", err)
+	}
+	want := `{}`
+	got := string(b)
+	if got != want {
+		t.Fatalf("Marshal: got %s want %s", got, want)
+	}
+}
+
+func TestStringBytes(t *testing.T) {
+	// Test that encodeState.stringBytes and encodeState.string use the same encoding.
+	es := &encodeState{}
+	var r []rune
+	for i := '\u0000'; i <= unicode.MaxRune; i++ {
+		r = append(r, i)
+	}
+	s := string(r) + "\xff\xff\xffhello" // some invalid UTF-8 too
+	_, err := es.string(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	esBytes := &encodeState{}
+	_, err = esBytes.stringBytes([]byte(s))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	enc := es.Buffer.String()
+	encBytes := esBytes.Buffer.String()
+	if enc != encBytes {
+		i := 0
+		for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] {
+			i++
+		}
+		enc = enc[i:]
+		encBytes = encBytes[i:]
+		i = 0
+		for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] {
+			i++
+		}
+		enc = enc[:len(enc)-i]
+		encBytes = encBytes[:len(encBytes)-i]
+
+		if len(enc) > 20 {
+			enc = enc[:20] + "..."
+		}
+		if len(encBytes) > 20 {
+			encBytes = encBytes[:20] + "..."
+		}
+
+		t.Errorf("encodings differ at %#q vs %#q", enc, encBytes)
+	}
+}

server/jjson/example_test.go

@@ -0,0 +1,83 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func ExampleMarshal() {
+	type ColorGroup struct {
+		ID     int
+		Name   string
+		Colors []string
+	}
+	group := ColorGroup{
+		ID:     1,
+		Name:   "Reds",
+		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
+	}
+	b, err := json.Marshal(group)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+	// Output:
+	// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
+}
+
+func ExampleUnmarshal() {
+	var jsonBlob = []byte(`[
+		{"Name": "Platypus", "Order": "Monotremata"},
+		{"Name": "Quoll",    "Order": "Dasyuromorphia"}
+	]`)
+	type Animal struct {
+		Name  string
+		Order string
+	}
+	var animals []Animal
+	err := json.Unmarshal(jsonBlob, &animals)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	fmt.Printf("%+v", animals)
+	// Output:
+	// [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}]
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder() {
+	const jsonStream = `
+		{"Name": "Ed", "Text": "Knock knock."}
+		{"Name": "Sam", "Text": "Who's there?"}
+		{"Name": "Ed", "Text": "Go fmt."}
+		{"Name": "Sam", "Text": "Go fmt who?"}
+		{"Name": "Ed", "Text": "Go fmt yourself!"}
+	`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		var m Message
+		if err := dec.Decode(&m); err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%s: %s\n", m.Name, m.Text)
+	}
+	// Output:
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+}

server/jjson/indent.go

@@ -0,0 +1,136 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import "bytes"
+
+// Compact appends to dst the JSON-encoded src with
+// insignificant space characters elided.
+func Compact(dst *bytes.Buffer, src []byte) error {
+	return compact(dst, src, false)
+}
+
+func compact(dst *bytes.Buffer, src []byte, escape bool) error {
+	origLen := dst.Len()
+	var scan scanner
+	scan.reset()
+	start := 0
+	for i, c := range src {
+		if escape && (c == '<' || c == '>' || c == '&') {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u00`)
+			dst.WriteByte(hex[c>>4])
+			dst.WriteByte(hex[c&0xF])
+			start = i + 1
+		}
+		// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
+		if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u202`)
+			dst.WriteByte(hex[src[i+2]&0xF])
+			start = i + 3
+		}
+		v := scan.step(&scan, int(c))
+		if v >= scanSkipSpace {
+			if v == scanError {
+				break
+			}
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			start = i + 1
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	if start < len(src) {
+		dst.Write(src[start:])
+	}
+	return nil
+}
+
+func newline(dst *bytes.Buffer, prefix, indent string, depth int) {
+	dst.WriteByte('\n')
+	dst.WriteString(prefix)
+	for i := 0; i < depth; i++ {
+		dst.WriteString(indent)
+	}
+}
+
+// Indent appends to dst an indented form of the JSON-encoded src.
+// Each element in a JSON object or array begins on a new,
+// indented line beginning with prefix followed by one or more
+// copies of indent according to the indentation nesting.
+// The data appended to dst has no trailing newline, to make it easier
+// to embed inside other formatted JSON data.
+func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
+	origLen := dst.Len()
+	var scan scanner
+	scan.reset()
+	needIndent := false
+	depth := 0
+	for _, c := range src {
+		scan.bytes++
+		v := scan.step(&scan, int(c))
+		if v == scanSkipSpace {
+			continue
+		}
+		if v == scanError {
+			break
+		}
+		if needIndent && v != scanEndObject && v != scanEndArray {
+			needIndent = false
+			depth++
+			newline(dst, prefix, indent, depth)
+		}
+
+		// Emit semantically uninteresting bytes
+		// (in particular, punctuation in strings) unmodified.
+		if v == scanContinue {
+			dst.WriteByte(c)
+			continue
+		}
+
+		// Add spacing around real punctuation.
+		switch c {
+		case '{', '[':
+			// delay indent so that empty object and array are formatted as {} and [].
+			needIndent = true
+			dst.WriteByte(c)
+
+		case ',':
+			dst.WriteByte(c)
+			newline(dst, prefix, indent, depth)
+
+		case ':':
+			dst.WriteByte(c)
+			dst.WriteByte(' ')
+
+		case '}', ']':
+			if needIndent {
+				// suppress indent in empty object/array
+				needIndent = false
+			} else {
+				depth--
+				newline(dst, prefix, indent, depth)
+			}
+			dst.WriteByte(c)
+
+		default:
+			dst.WriteByte(c)
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	return nil
+}

server/jjson/scanner.go

@@ -0,0 +1,623 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+// JSON value parser state machine.
+// Just about at the limit of what is reasonable to write by hand.
+// Some parts are a bit tedious, but overall it nicely factors out the
+// otherwise common code from the multiple scanning functions
+// in this package (Compact, Indent, checkValid, nextValue, etc).
+//
+// This file starts with two simple examples using the scanner
+// before diving into the scanner itself.
+
+import "strconv"
+
+// checkValid verifies that data is valid JSON-encoded data.
+// scan is passed in for use by checkValid to avoid an allocation.
+func checkValid(data []byte, scan *scanner) error {
+	scan.reset()
+	for _, c := range data {
+		scan.bytes++
+		if scan.step(scan, int(c)) == scanError {
+			return scan.err
+		}
+	}
+	if scan.eof() == scanError {
+		return scan.err
+	}
+	return nil
+}
+
+// nextValue splits data after the next whole JSON value,
+// returning that value and the bytes that follow it as separate slices.
+// scan is passed in for use by nextValue to avoid an allocation.
+func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) {
+	scan.reset()
+	for i, c := range data {
+		v := scan.step(scan, int(c))
+		if v >= scanEnd {
+			switch v {
+			case scanError:
+				return nil, nil, scan.err
+			case scanEnd:
+				return data[0:i], data[i:], nil
+			}
+		}
+	}
+	if scan.eof() == scanError {
+		return nil, nil, scan.err
+	}
+	return data, nil, nil
+}
+
+// A SyntaxError is a description of a JSON syntax error.
+type SyntaxError struct {
+	msg    string // description of error
+	Offset int64  // error occurred after reading Offset bytes
+}
+
+func (e *SyntaxError) Error() string { return e.msg }
+
+// A scanner is a JSON scanning state machine.
+// Callers call scan.reset() and then pass bytes in one at a time
+// by calling scan.step(&scan, c) for each byte.
+// The return value, referred to as an opcode, tells the
+// caller about significant parsing events like beginning
+// and ending literals, objects, and arrays, so that the
+// caller can follow along if it wishes.
+// The return value scanEnd indicates that a single top-level
+// JSON value has been completed, *before* the byte that
+// just got passed in.  (The indication must be delayed in order
+// to recognize the end of numbers: is 123 a whole value or
+// the beginning of 12345e+6?).
+type scanner struct {
+	// The step is a func to be called to execute the next transition.
+	// Also tried using an integer constant and a single func
+	// with a switch, but using the func directly was 10% faster
+	// on a 64-bit Mac Mini, and it's nicer to read.
+	step func(*scanner, int) int
+
+	// Reached end of top-level value.
+	endTop bool
+
+	// Stack of what we're in the middle of - array values, object keys, object values.
+	parseState []int
+
+	// Error that happened, if any.
+	err error
+
+	// 1-byte redo (see undo method)
+	redo      bool
+	redoCode  int
+	redoState func(*scanner, int) int
+
+	// total bytes consumed, updated by decoder.Decode
+	bytes int64
+}
+
+// These values are returned by the state transition functions
+// assigned to scanner.state and the method scanner.eof.
+// They give details about the current state of the scan that
+// callers might be interested to know about.
+// It is okay to ignore the return value of any particular
+// call to scanner.state: if one call returns scanError,
+// every subsequent call will return scanError too.
+const (
+	// Continue.
+	scanContinue     = iota // uninteresting byte
+	scanBeginLiteral        // end implied by next result != scanContinue
+	scanBeginObject         // begin object
+	scanObjectKey           // just finished object key (string)
+	scanObjectValue         // just finished non-last object value
+	scanEndObject           // end object (implies scanObjectValue if possible)
+	scanBeginArray          // begin array
+	scanArrayValue          // just finished array value
+	scanEndArray            // end array (implies scanArrayValue if possible)
+	scanSkipSpace           // space byte; can skip; known to be last "continue" result
+
+	// Stop.
+	scanEnd   // top-level value ended *before* this byte; known to be first "stop" result
+	scanError // hit an error, scanner.err.
+)
+
+// These values are stored in the parseState stack.
+// They give the current state of a composite value
+// being scanned.  If the parser is inside a nested value
+// the parseState describes the nested state, outermost at entry 0.
+const (
+	parseObjectKey   = iota // parsing object key (before colon)
+	parseObjectValue        // parsing object value (after colon)
+	parseArrayValue         // parsing array value
+)
+
+// reset prepares the scanner for use.
+// It must be called before calling s.step.
+func (s *scanner) reset() {
+	s.step = stateBeginValue
+	s.parseState = s.parseState[0:0]
+	s.err = nil
+	s.redo = false
+	s.endTop = false
+}
+
+// eof tells the scanner that the end of input has been reached.
+// It returns a scan status just as s.step does.
+func (s *scanner) eof() int {
+	if s.err != nil {
+		return scanError
+	}
+	if s.endTop {
+		return scanEnd
+	}
+	s.step(s, ' ')
+	if s.endTop {
+		return scanEnd
+	}
+	if s.err == nil {
+		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	}
+	return scanError
+}
+
+// pushParseState pushes a new parse state p onto the parse stack.
+func (s *scanner) pushParseState(p int) {
+	s.parseState = append(s.parseState, p)
+}
+
+// popParseState pops a parse state (already obtained) off the stack
+// and updates s.step accordingly.
+func (s *scanner) popParseState() {
+	n := len(s.parseState) - 1
+	s.parseState = s.parseState[0:n]
+	s.redo = false
+	if n == 0 {
+		s.step = stateEndTop
+		s.endTop = true
+	} else {
+		s.step = stateEndValue
+	}
+}
+
+func isSpace(c rune) bool {
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
+}
+
+// stateBeginValueOrEmpty is the state after reading `[`.
+func stateBeginValueOrEmpty(s *scanner, c int) int {
+	if c <= ' ' && isSpace(rune(c)) {
+		return scanSkipSpace
+	}
+	if c == ']' {
+		return stateEndValue(s, c)
+	}
+	return stateBeginValue(s, c)
+}
+
+// stateBeginValue is the state at the beginning of the input.
+func stateBeginValue(s *scanner, c int) int {
+	if c <= ' ' && isSpace(rune(c)) {
+		return scanSkipSpace
+	}
+	switch c {
+	case '{':
+		s.step = stateBeginStringOrEmpty
+		s.pushParseState(parseObjectKey)
+		return scanBeginObject
+	case '[':
+		s.step = stateBeginValueOrEmpty
+		s.pushParseState(parseArrayValue)
+		return scanBeginArray
+	case '"':
+		s.step = stateInString
+		return scanBeginLiteral
+	case '-':
+		s.step = stateNeg
+		return scanBeginLiteral
+	case '0': // beginning of 0.123
+		s.step = state0
+		return scanBeginLiteral
+	case 't': // beginning of true
+		s.step = stateT
+		return scanBeginLiteral
+	case 'f': // beginning of false
+		s.step = stateF
+		return scanBeginLiteral
+	case 'n': // beginning of null
+		s.step = stateN
+		return scanBeginLiteral
+	}
+	if '1' <= c && c <= '9' { // beginning of 1234.5
+		s.step = state1
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of value")
+}
+
+// stateBeginStringOrEmpty is the state after reading `{`.
+func stateBeginStringOrEmpty(s *scanner, c int) int {
+	if c <= ' ' && isSpace(rune(c)) {
+		return scanSkipSpace
+	}
+	if c == '}' {
+		n := len(s.parseState)
+		s.parseState[n-1] = parseObjectValue
+		return stateEndValue(s, c)
+	}
+	return stateBeginString(s, c)
+}
+
+// stateBeginString is the state after reading `{"key": value,`.
+func stateBeginString(s *scanner, c int) int {
+	if c <= ' ' && isSpace(rune(c)) {
+		return scanSkipSpace
+	}
+	if c == '"' {
+		s.step = stateInString
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of object key string")
+}
+
+// stateEndValue is the state after completing a value,
+// such as after reading `{}` or `true` or `["x"`.
+func stateEndValue(s *scanner, c int) int {
+	n := len(s.parseState)
+	if n == 0 {
+		// Completed top-level before the current byte.
+		s.step = stateEndTop
+		s.endTop = true
+		return stateEndTop(s, c)
+	}
+	if c <= ' ' && isSpace(rune(c)) {
+		s.step = stateEndValue
+		return scanSkipSpace
+	}
+	ps := s.parseState[n-1]
+	switch ps {
+	case parseObjectKey:
+		if c == ':' {
+			s.parseState[n-1] = parseObjectValue
+			s.step = stateBeginValue
+			return scanObjectKey
+		}
+		return s.error(c, "after object key")
+	case parseObjectValue:
+		if c == ',' {
+			s.parseState[n-1] = parseObjectKey
+			s.step = stateBeginString
+			return scanObjectValue
+		}
+		if c == '}' {
+			s.popParseState()
+			return scanEndObject
+		}
+		return s.error(c, "after object key:value pair")
+	case parseArrayValue:
+		if c == ',' {
+			s.step = stateBeginValue
+			return scanArrayValue
+		}
+		if c == ']' {
+			s.popParseState()
+			return scanEndArray
+		}
+		return s.error(c, "after array element")
+	}
+	return s.error(c, "")
+}
+
+// stateEndTop is the state after finishing the top-level value,
+// such as after reading `{}` or `[1,2,3]`.
+// Only space characters should be seen now.
+func stateEndTop(s *scanner, c int) int {
+	if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
+		// Complain about non-space byte on next call.
+		s.error(c, "after top-level value")
+	}
+	return scanEnd
+}
+
+// stateInString is the state after reading `"`.
+func stateInString(s *scanner, c int) int {
+	if c == '"' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	if c == '\\' {
+		s.step = stateInStringEsc
+		return scanContinue
+	}
+	if c < 0x20 {
+		return s.error(c, "in string literal")
+	}
+	return scanContinue
+}
+
+// stateInStringEsc is the state after reading `"\` during a quoted string.
+func stateInStringEsc(s *scanner, c int) int {
+	switch c {
+	case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
+		s.step = stateInString
+		return scanContinue
+	}
+	if c == 'u' {
+		s.step = stateInStringEscU
+		return scanContinue
+	}
+	return s.error(c, "in string escape code")
+}
+
+// stateInStringEscU is the state after reading `"\u` during a quoted string.
+func stateInStringEscU(s *scanner, c int) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU1
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
+func stateInStringEscU1(s *scanner, c int) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU12
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
+func stateInStringEscU12(s *scanner, c int) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU123
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
+func stateInStringEscU123(s *scanner, c int) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInString
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateNeg is the state after reading `-` during a number.
+func stateNeg(s *scanner, c int) int {
+	if c == '0' {
+		s.step = state0
+		return scanContinue
+	}
+	if '1' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return s.error(c, "in numeric literal")
+}
+
+// state1 is the state after reading a non-zero integer during a number,
+// such as after reading `1` or `100` but not `0`.
+func state1(s *scanner, c int) int {
+	if '0' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return state0(s, c)
+}
+
+// state0 is the state after reading `0` during a number.
+func state0(s *scanner, c int) int {
+	if c == '.' {
+		s.step = stateDot
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateDot is the state after reading the integer and decimal point in a number,
+// such as after reading `1.`.
+func stateDot(s *scanner, c int) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	return s.error(c, "after decimal point in numeric literal")
+}
+
+// stateDot0 is the state after reading the integer, decimal point, and subsequent
+// digits of a number, such as after reading `3.14`.
+func stateDot0(s *scanner, c int) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateE is the state after reading the mantissa and e in a number,
+// such as after reading `314e` or `0.314e`.
+func stateE(s *scanner, c int) int {
+	if c == '+' {
+		s.step = stateESign
+		return scanContinue
+	}
+	if c == '-' {
+		s.step = stateESign
+		return scanContinue
+	}
+	return stateESign(s, c)
+}
+
+// stateESign is the state after reading the mantissa, e, and sign in a number,
+// such as after reading `314e-` or `0.314e+`.
+func stateESign(s *scanner, c int) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return s.error(c, "in exponent of numeric literal")
+}
+
+// stateE0 is the state after reading the mantissa, e, optional sign,
+// and at least one digit of the exponent in a number,
+// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
+func stateE0(s *scanner, c int) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateT is the state after reading `t`.
+func stateT(s *scanner, c int) int {
+	if c == 'r' {
+		s.step = stateTr
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'r')")
+}
+
+// stateTr is the state after reading `tr`.
+func stateTr(s *scanner, c int) int {
+	if c == 'u' {
+		s.step = stateTru
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'u')")
+}
+
+// stateTru is the state after reading `tru`.
+func stateTru(s *scanner, c int) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'e')")
+}
+
+// stateF is the state after reading `f`.
+func stateF(s *scanner, c int) int {
+	if c == 'a' {
+		s.step = stateFa
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'a')")
+}
+
+// stateFa is the state after reading `fa`.
+func stateFa(s *scanner, c int) int {
+	if c == 'l' {
+		s.step = stateFal
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'l')")
+}
+
+// stateFal is the state after reading `fal`.
+func stateFal(s *scanner, c int) int {
+	if c == 's' {
+		s.step = stateFals
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 's')")
+}
+
+// stateFals is the state after reading `fals`.
+func stateFals(s *scanner, c int) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'e')")
+}
+
+// stateN is the state after reading `n`.
+func stateN(s *scanner, c int) int {
+	if c == 'u' {
+		s.step = stateNu
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'u')")
+}
+
+// stateNu is the state after reading `nu`.
+func stateNu(s *scanner, c int) int {
+	if c == 'l' {
+		s.step = stateNul
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateNul is the state after reading `nul`.
+func stateNul(s *scanner, c int) int {
+	if c == 'l' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateError is the state after reaching a syntax error,
+// such as after reading `[1}` or `5.1.2`.
+func stateError(s *scanner, c int) int {
+	return scanError
+}
+
+// error records an error and switches to the error state.
+func (s *scanner) error(c int, context string) int {
+	s.step = stateError
+	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	return scanError
+}
+
+// quoteChar formats c as a quoted character literal
+func quoteChar(c int) string {
+	// special cases - different from quoted strings
+	if c == '\'' {
+		return `'\''`
+	}
+	if c == '"' {
+		return `'"'`
+	}
+
+	// use quoted string with different quotation marks
+	s := strconv.Quote(string(c))
+	return "'" + s[1:len(s)-1] + "'"
+}
+
+// undo causes the scanner to return scanCode from the next state transition.
+// This gives callers a simple 1-byte undo mechanism.
+func (s *scanner) undo(scanCode int) {
+	if s.redo {
+		panic("json: invalid use of scanner")
+	}
+	s.redoCode = scanCode
+	s.redoState = s.step
+	s.step = stateRedo
+	s.redo = true
+}
+
+// stateRedo helps implement the scanner's 1-byte undo.
+func stateRedo(s *scanner, c int) int {
+	s.redo = false
+	s.step = s.redoState
+	return s.redoCode
+}

server/jjson/scanner_test.go

@@ -0,0 +1,319 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+// Tests of simple examples.
+
+type example struct {
+	compact string
+	indent  string
+}
+
+var examples = []example{
+	{`1`, `1`},
+	{`{}`, `{}`},
+	{`[]`, `[]`},
+	{`{"":2}`, "{\n\t\"\": 2\n}"},
+	{`[3]`, "[\n\t3\n]"},
+	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
+	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
+	{ex1, ex1i},
+}
+
+var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`
+
+var ex1i = `[
+	true,
+	false,
+	null,
+	"x",
+	1,
+	1.5,
+	0,
+	-5e+2
+]`
+
+func TestCompact(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.compact)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.compact, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
+		}
+
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.indent)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.indent, err)
+			continue
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
+		}
+	}
+}
+
+func TestCompactSeparators(t *testing.T) {
+	// U+2028 and U+2029 should be escaped inside strings.
+	// They should not appear outside strings.
+	tests := []struct {
+		in, compact string
+	}{
+		{"{\"\u2028\": 1}", `{"\u2028":1}`},
+		{"{\"\u2029\" :2}", `{"\u2029":2}`},
+	}
+	for _, tt := range tests {
+		var buf bytes.Buffer
+		if err := Compact(&buf, []byte(tt.in)); err != nil {
+			t.Errorf("Compact(%q): %v", tt.in, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%q) = %q, want %q", tt.in, s, tt.compact)
+		}
+	}
+}
+
+func TestIndent(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.indent, err)
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
+		}
+
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.compact, err)
+			continue
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
+		}
+	}
+}
+
+// Tests of a large random structure.
+
+func TestCompactBig(t *testing.T) {
+	initBig()
+	var buf bytes.Buffer
+	if err := Compact(&buf, jsonBig); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b := buf.Bytes()
+	if !bytes.Equal(b, jsonBig) {
+		t.Error("Compact(jsonBig) != jsonBig")
+		diff(t, b, jsonBig)
+		return
+	}
+}
+
+func TestIndentBig(t *testing.T) {
+	initBig()
+	var buf bytes.Buffer
+	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
+		t.Fatalf("Indent1: %v", err)
+	}
+	b := buf.Bytes()
+	if len(b) == len(jsonBig) {
+		// jsonBig is compact (no unnecessary spaces);
+		// indenting should make it bigger
+		t.Fatalf("Indent(jsonBig) did not get bigger")
+	}
+
+	// should be idempotent
+	var buf1 bytes.Buffer
+	if err := Indent(&buf1, b, "", "\t"); err != nil {
+		t.Fatalf("Indent2: %v", err)
+	}
+	b1 := buf1.Bytes()
+	if !bytes.Equal(b1, b) {
+		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
+		diff(t, b1, b)
+		return
+	}
+
+	// should get back to original
+	buf1.Reset()
+	if err := Compact(&buf1, b); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b1 = buf1.Bytes()
+	if !bytes.Equal(b1, jsonBig) {
+		t.Error("Compact(Indent(jsonBig)) != jsonBig")
+		diff(t, b1, jsonBig)
+		return
+	}
+}
+
+type indentErrorTest struct {
+	in  string
+	err error
+}
+
+var indentErrorTests = []indentErrorTest{
+	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
+	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+}
+
+func TestIndentErrors(t *testing.T) {
+	for i, tt := range indentErrorTests {
+		slice := make([]uint8, 0)
+		buf := bytes.NewBuffer(slice)
+		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
+			if !reflect.DeepEqual(err, tt.err) {
+				t.Errorf("#%d: Indent: %#v", i, err)
+				continue
+			}
+		}
+	}
+}
+
+func TestNextValueBig(t *testing.T) {
+	initBig()
+	var scan scanner
+	item, rest, err := nextValue(jsonBig, &scan)
+	if err != nil {
+		t.Fatalf("nextValue: %s", err)
+	}
+	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
+		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
+	}
+	if len(rest) != 0 {
+		t.Errorf("invalid rest: %d", len(rest))
+	}
+
+	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
+	if err != nil {
+		t.Fatalf("nextValue extra: %s", err)
+	}
+	if len(item) != len(jsonBig) {
+		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
+	}
+	if string(rest) != "HELLO WORLD" {
+		t.Errorf("invalid rest: %d", len(rest))
+	}
+}
+
+var benchScan scanner
+
+func BenchmarkSkipValue(b *testing.B) {
+	initBig()
+	for i := 0; i < b.N; i++ {
+		nextValue(jsonBig, &benchScan)
+	}
+	b.SetBytes(int64(len(jsonBig)))
+}
+
+func diff(t *testing.T, a, b []byte) {
+	for i := 0; ; i++ {
+		if i >= len(a) || i >= len(b) || a[i] != b[i] {
+			j := i - 10
+			if j < 0 {
+				j = 0
+			}
+			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
+			return
+		}
+	}
+}
+
+func trim(b []byte) []byte {
+	if len(b) > 20 {
+		return b[0:20]
+	}
+	return b
+}
+
+// Generate a random JSON object.
+
+var jsonBig []byte
+
+const (
+	big   = 10000
+	small = 100
+)
+
+func initBig() {
+	n := big
+	if testing.Short() {
+		n = small
+	}
+	if len(jsonBig) != n {
+		b, err := Marshal(genValue(n))
+		if err != nil {
+			panic(err)
+		}
+		jsonBig = b
+	}
+}
+
+func genValue(n int) interface{} {
+	if n > 1 {
+		switch rand.Intn(2) {
+		case 0:
+			return genArray(n)
+		case 1:
+			return genMap(n)
+		}
+	}
+	switch rand.Intn(3) {
+	case 0:
+		return rand.Intn(2) == 0
+	case 1:
+		return rand.NormFloat64()
+	case 2:
+		return genString(30)
+	}
+	panic("unreachable")
+}
+
+func genString(stddev float64) string {
+	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
+	c := make([]rune, n)
+	for i := range c {
+		f := math.Abs(rand.NormFloat64()*64 + 32)
+		if f > 0x10ffff {
+			f = 0x10ffff
+		}
+		c[i] = rune(f)
+	}
+	return string(c)
+}
+
+func genArray(n int) []interface{} {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	x := make([]interface{}, f)
+	for i := range x {
+		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}
+
+func genMap(n int) map[string]interface{} {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	if n > 0 && f == 0 {
+		f = 1
+	}
+	x := make(map[string]interface{})
+	for i := 0; i < f; i++ {
+		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}

server/jjson/stream.go

@@ -0,0 +1,200 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+// A Decoder reads and decodes JSON objects from an input stream.
+type Decoder struct {
+	r    io.Reader
+	buf  []byte
+	d    decodeState
+	scan scanner
+	err  error
+}
+
+// NewDecoder returns a new decoder that reads from r.
+//
+// The decoder introduces its own buffering and may
+// read data from r beyond the JSON values requested.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{r: r}
+}
+
+// UseNumber causes the Decoder to unmarshal a number into an interface{} as a
+// Number instead of as a float64.
+func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
+
+// Decode reads the next JSON-encoded value from its
+// input and stores it in the value pointed to by v.
+//
+// See the documentation for Unmarshal for details about
+// the conversion of JSON into a Go value.
+func (dec *Decoder) Decode(v interface{}) error {
+	if dec.err != nil {
+		return dec.err
+	}
+
+	n, err := dec.readValue()
+	if err != nil {
+		return err
+	}
+
+	// Don't save err from unmarshal into dec.err:
+	// the connection is still usable since we read a complete JSON
+	// object from it before the error happened.
+	dec.d.init(dec.buf[0:n])
+	err = dec.d.unmarshal(v)
+
+	// Slide rest of data down.
+	rest := copy(dec.buf, dec.buf[n:])
+	dec.buf = dec.buf[0:rest]
+
+	return err
+}
+
+// Buffered returns a reader of the data remaining in the Decoder's
+// buffer. The reader is valid until the next call to Decode.
+func (dec *Decoder) Buffered() io.Reader {
+	return bytes.NewReader(dec.buf)
+}
+
+// readValue reads a JSON value into dec.buf.
+// It returns the length of the encoding.
+func (dec *Decoder) readValue() (int, error) {
+	dec.scan.reset()
+
+	scanp := 0
+	var err error
+Input:
+	for {
+		// Look in the buffer for a new value.
+		for i, c := range dec.buf[scanp:] {
+			dec.scan.bytes++
+			v := dec.scan.step(&dec.scan, int(c))
+			if v == scanEnd {
+				scanp += i
+				break Input
+			}
+			// scanEnd is delayed one byte.
+			// We might block trying to get that byte from src,
+			// so instead invent a space byte.
+			if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd {
+				scanp += i + 1
+				break Input
+			}
+			if v == scanError {
+				dec.err = dec.scan.err
+				return 0, dec.scan.err
+			}
+		}
+		scanp = len(dec.buf)
+
+		// Did the last read have an error?
+		// Delayed until now to allow buffer scan.
+		if err != nil {
+			if err == io.EOF {
+				if dec.scan.step(&dec.scan, ' ') == scanEnd {
+					break Input
+				}
+				if nonSpace(dec.buf) {
+					err = io.ErrUnexpectedEOF
+				}
+			}
+			dec.err = err
+			return 0, err
+		}
+
+		// Make room to read more into the buffer.
+		const minRead = 512
+		if cap(dec.buf)-len(dec.buf) < minRead {
+			newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead)
+			copy(newBuf, dec.buf)
+			dec.buf = newBuf
+		}
+
+		// Read.  Delay error for next iteration (after scan).
+		var n int
+		n, err = dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
+		dec.buf = dec.buf[0 : len(dec.buf)+n]
+	}
+	return scanp, nil
+}
+
+func nonSpace(b []byte) bool {
+	for _, c := range b {
+		if !isSpace(rune(c)) {
+			return true
+		}
+	}
+	return false
+}
+
+// An Encoder writes JSON objects to an output stream.
+type Encoder struct {
+	w   io.Writer
+	e   encodeState
+	err error
+}
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{w: w}
+}
+
+// Encode writes the JSON encoding of v to the connection.
+//
+// See the documentation for Marshal for details about the
+// conversion of Go values to JSON.
+func (enc *Encoder) Encode(v interface{}) error {
+	if enc.err != nil {
+		return enc.err
+	}
+	e := newEncodeState()
+	err := e.marshal(v)
+	if err != nil {
+		return err
+	}
+
+	// Terminate each value with a newline.
+	// This makes the output look a little nicer
+	// when debugging, and some kind of space
+	// is required if the encoded value was a number,
+	// so that the reader knows there aren't more
+	// digits coming.
+	e.WriteByte('\n')
+
+	if _, err = enc.w.Write(e.Bytes()); err != nil {
+		enc.err = err
+	}
+	putEncodeState(e)
+	return err
+}
+
+// RawMessage is a raw encoded JSON object.
+// It implements Marshaler and Unmarshaler and can
+// be used to delay JSON decoding or precompute a JSON encoding.
+type RawMessage []byte
+
+// MarshalJSON returns *m as the JSON encoding of m.
+func (m *RawMessage) MarshalJSON() ([]byte, error) {
+	return *m, nil
+}
+
+// UnmarshalJSON sets *m to a copy of data.
+func (m *RawMessage) UnmarshalJSON(data []byte) error {
+	if m == nil {
+		return errors.New("json.RawMessage: UnmarshalJSON on nil pointer")
+	}
+	*m = append((*m)[0:0], data...)
+	return nil
+}
+
+var _ Marshaler = (*RawMessage)(nil)
+var _ Unmarshaler = (*RawMessage)(nil)

server/jjson/stream_test.go

@@ -0,0 +1,206 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Test values for the stream test.
+// One of each JSON kind.
+var streamTest = []interface{}{
+	0.1,
+	"hello",
+	nil,
+	true,
+	false,
+	[]interface{}{"a", "b", "c"},
+	map[string]interface{}{"K": "Kelvin", "ß": "long s"},
+	3.14, // another value to make sure something can follow map
+}
+
+var streamEncoded = `0.1
+"hello"
+null
+true
+false
+["a","b","c"]
+{"ß":"long s","K":"Kelvin"}
+3.14
+`
+
+func TestEncoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		var buf bytes.Buffer
+		enc := NewEncoder(&buf)
+		for j, v := range streamTest[0:i] {
+			if err := enc.Encode(v); err != nil {
+				t.Fatalf("encode #%d: %v", j, err)
+			}
+		}
+		if have, want := buf.String(), nlines(streamEncoded, i); have != want {
+			t.Errorf("encoding %d items: mismatch", i)
+			diff(t, []byte(have), []byte(want))
+			break
+		}
+	}
+}
+
+func TestDecoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		// Use stream without newlines as input,
+		// just to stress the decoder even more.
+		// Our test input does not include back-to-back numbers.
+		// Otherwise stripping the newlines would
+		// merge two adjacent JSON values.
+		var buf bytes.Buffer
+		for _, c := range nlines(streamEncoded, i) {
+			if c != '\n' {
+				buf.WriteRune(c)
+			}
+		}
+		out := make([]interface{}, i)
+		dec := NewDecoder(&buf)
+		for j := range out {
+			if err := dec.Decode(&out[j]); err != nil {
+				t.Fatalf("decode #%d/%d: %v", j, i, err)
+			}
+		}
+		if !reflect.DeepEqual(out, streamTest[0:i]) {
+			t.Errorf("decoding %d items: mismatch", i)
+			for j := range out {
+				if !reflect.DeepEqual(out[j], streamTest[j]) {
+					t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
+				}
+			}
+			break
+		}
+	}
+}
+
+func TestDecoderBuffered(t *testing.T) {
+	r := strings.NewReader(`{"Name": "Gopher"} extra `)
+	var m struct {
+		Name string
+	}
+	d := NewDecoder(r)
+	err := d.Decode(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m.Name != "Gopher" {
+		t.Errorf("Name = %q; want Gopher", m.Name)
+	}
+	rest, err := ioutil.ReadAll(d.Buffered())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g, w := string(rest), " extra "; g != w {
+		t.Errorf("Remaining = %q; want %q", g, w)
+	}
+}
+
+func nlines(s string, n int) string {
+	if n <= 0 {
+		return ""
+	}
+	for i, c := range s {
+		if c == '\n' {
+			if n--; n == 0 {
+				return s[0 : i+1]
+			}
+		}
+	}
+	return s
+}
+
+func TestRawMessage(t *testing.T) {
+	// TODO(rsc): Should not need the * in *RawMessage
+	var data struct {
+		X  float64
+		Id *RawMessage
+		Y  float32
+	}
+	const raw = `["\u0056",null]`
+	const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if string([]byte(*data.Id)) != raw {
+		t.Fatalf("Raw mismatch: have %#q want %#q", []byte(*data.Id), raw)
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+func TestNullRawMessage(t *testing.T) {
+	// TODO(rsc): Should not need the * in *RawMessage
+	var data struct {
+		X  float64
+		Id *RawMessage
+		Y  float32
+	}
+	data.Id = new(RawMessage)
+	const msg = `{"X":0.1,"Id":null,"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if data.Id != nil {
+		t.Fatalf("Raw mismatch: have non-nil, want nil")
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+var blockingTests = []string{
+	`{"x": 1}`,
+	`[1, 2, 3]`,
+}
+
+func TestBlocking(t *testing.T) {
+	for _, enc := range blockingTests {
+		r, w := net.Pipe()
+		go w.Write([]byte(enc))
+		var val interface{}
+
+		// If Decode reads beyond what w.Write writes above,
+		// it will block, and the test will deadlock.
+		if err := NewDecoder(r).Decode(&val); err != nil {
+			t.Errorf("decoding %s: %v", enc, err)
+		}
+		r.Close()
+		w.Close()
+	}
+}
+
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	for i := 0; i < b.N; i++ {
+		if err := NewEncoder(ioutil.Discard).Encode(v); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

server/jjson/tagkey_test.go

@@ -0,0 +1,115 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"testing"
+)
+
+type basicLatin2xTag struct {
+	V string `json:"$%-/"`
+}
+
+type basicLatin3xTag struct {
+	V string `json:"0123456789"`
+}
+
+type basicLatin4xTag struct {
+	V string `json:"ABCDEFGHIJKLMO"`
+}
+
+type basicLatin5xTag struct {
+	V string `json:"PQRSTUVWXYZ_"`
+}
+
+type basicLatin6xTag struct {
+	V string `json:"abcdefghijklmno"`
+}
+
+type basicLatin7xTag struct {
+	V string `json:"pqrstuvwxyz"`
+}
+
+type miscPlaneTag struct {
+	V string `json:"色は匂へど"`
+}
+
+type percentSlashTag struct {
+	V string `json:"text/html%"` // http://golang.org/issue/2718
+}
+
+type punctuationTag struct {
+	V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // http://golang.org/issue/3546
+}
+
+type emptyTag struct {
+	W string
+}
+
+type misnamedTag struct {
+	X string `jsom:"Misnamed"`
+}
+
+type badFormatTag struct {
+	Y string `:"BadFormat"`
+}
+
+type badCodeTag struct {
+	Z string `json:" !\"#&'()*+,."`
+}
+
+type spaceTag struct {
+	Q string `json:"With space"`
+}
+
+type unicodeTag struct {
+	W string `json:"Ελλάδα"`
+}
+
+var structTagObjectKeyTests = []struct {
+	raw   interface{}
+	value string
+	key   string
+}{
+	{basicLatin2xTag{"2x"}, "2x", "$%-/"},
+	{basicLatin3xTag{"3x"}, "3x", "0123456789"},
+	{basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"},
+	{basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"},
+	{basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"},
+	{basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"},
+	{miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"},
+	{emptyTag{"Pour Moi"}, "Pour Moi", "W"},
+	{misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"},
+	{badFormatTag{"Orfevre"}, "Orfevre", "Y"},
+	{badCodeTag{"Reliable Man"}, "Reliable Man", "Z"},
+	{percentSlashTag{"brut"}, "brut", "text/html%"},
+	{punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:<=>?@[]^_{|}~"},
+	{spaceTag{"Perreddu"}, "Perreddu", "With space"},
+	{unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"},
+}
+
+func TestStructTagObjectKey(t *testing.T) {
+	for _, tt := range structTagObjectKeyTests {
+		b, err := Marshal(tt.raw)
+		if err != nil {
+			t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err)
+		}
+		var f interface{}
+		err = Unmarshal(b, &f)
+		if err != nil {
+			t.Fatalf("Unmarshal(%#q) failed: %v", b, err)
+		}
+		for i, v := range f.(map[string]interface{}) {
+			switch i {
+			case tt.key:
+				if s, ok := v.(string); !ok || s != tt.value {
+					t.Fatalf("Unexpected value: %#q, want %v", s, tt.value)
+				}
+			default:
+				t.Fatalf("Unexpected key: %#q, from %#q", i, b)
+			}
+		}
+	}
+}

server/jjson/tags.go

@@ -0,0 +1,44 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"strings"
+)
+
+// tagOptions is the string following a comma in a struct field's "json"
+// tag, or the empty string. It does not include the leading comma.
+type tagOptions string
+
+// parseTag splits a struct field's json tag into its name and
+// comma-separated options.
+func parseTag(tag string) (string, tagOptions) {
+	if idx := strings.Index(tag, ","); idx != -1 {
+		return tag[:idx], tagOptions(tag[idx+1:])
+	}
+	return tag, tagOptions("")
+}
+
+// Contains reports whether a comma-separated list of options
+// contains a particular substr flag. substr must be surrounded by a
+// string boundary or commas.
+func (o tagOptions) Contains(optionName string) bool {
+	if len(o) == 0 {
+		return false
+	}
+	s := string(o)
+	for s != "" {
+		var next string
+		i := strings.Index(s, ",")
+		if i >= 0 {
+			s, next = s[:i], s[i+1:]
+		}
+		if s == optionName {
+			return true
+		}
+		s = next
+	}
+	return false
+}

server/jjson/tags_test.go

@@ -0,0 +1,28 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jjson
+
+import (
+	"testing"
+)
+
+func TestTagParsing(t *testing.T) {
+	name, opts := parseTag("field,foobar,foo")
+	if name != "field" {
+		t.Fatalf("name = %q, want field", name)
+	}
+	for _, tt := range []struct {
+		opt  string
+		want bool
+	}{
+		{"foobar", true},
+		{"foo", true},
+		{"bar", false},
+	} {
+		if opts.Contains(tt.opt) != tt.want {
+			t.Errorf("Contains(%q) = %v", tt.opt, !tt.want)
+		}
+	}
+}

server/log/log.go

@@ -0,0 +1,67 @@
+package log
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+var logLevel int = LOG_LEVEL_DEBUG
+var numErrors int64 = 0
+
+const (
+	LOG_LEVEL_ERROR   = 0
+	LOG_LEVEL_RECORD  = 1
+	LOG_LEVEL_WARNING = 2
+	LOG_LEVEL_NOTICE  = 3
+	LOG_LEVEL_DEBUG   = 4
+)
+
+var prefices = map[int]string{
+	LOG_LEVEL_RECORD:  "record  : ",
+	LOG_LEVEL_ERROR:   "error   : ",
+	LOG_LEVEL_WARNING: "warning : ",
+	LOG_LEVEL_NOTICE:  "notice  : ",
+	LOG_LEVEL_DEBUG:   "debug   : ",
+}
+
+func log(msg string, level int) string {
+	if level <= logLevel {
+		prefix := time.Now().Format(time.RFC3339) + " " + prefices[level]
+		lines := strings.Split(msg, "\n")
+		for i := 0; i < len(lines); i++ {
+			fmt.Println(prefix + lines[i])
+		}
+	}
+	return msg
+}
+
+func SetLogLevel(level int) bool {
+	if level > LOG_LEVEL_ERROR && level <= LOG_LEVEL_DEBUG {
+		logLevel = level
+		return true
+	} else {
+		return false
+	}
+}
+
+func Debug(msg string) string {
+	return log(msg, LOG_LEVEL_DEBUG)
+}
+
+func Notice(msg string) string {
+	return log(msg, LOG_LEVEL_NOTICE)
+}
+
+func Warning(msg string) string {
+	return log(msg, LOG_LEVEL_WARNING)
+}
+
+func Record(msg string) string {
+	return log(msg, LOG_LEVEL_RECORD)
+}
+
+func Error(msg string) string {
+	numErrors++
+	return log(fmt.Sprintf("(%d) ", numErrors)+msg, LOG_LEVEL_ERROR)
+}

server/repo/content/constants.go

@@ -1,5 +1,6 @@
 package content
 
 const (
-	INDENT string = "\t"
+	INDENT         string = "\t"
+	PATH_SEPARATOR        = "/"
 )

server/repo/content/repoNode.go

@@ -16,6 +16,7 @@ type RepoNode struct {
 	Groups   []string                     `json:"groups"`
 	Data     map[string]interface{}       `json:"data"`
 	Content  map[string]interface{}       `json:"content"`
+	Index    []string                     `json:"index"`
 	Nodes    map[string]*RepoNode         `json:"nodes"`
 	LinkIds  map[string]map[string]string `json:"linkIds"` // ids to link to
 	parent   *RepoNode

server/repo/content/siteContent.go

@@ -8,6 +8,7 @@ const (
 
 type SiteContent struct {
 	Status   int               `json:"status"`
+	URI      string            `json:"URI"`
 	Region   string            `json:"region"`
 	Language string            `json:"language"`
 	Item     *Item             `json:"item"`

server/repo/repo.go

@@ -1,10 +1,11 @@
 package repo
 
 import (
-	"fmt"
+	"github.com/foomo/ContentServer/server/log"
 	"github.com/foomo/ContentServer/server/repo/content"
 	"github.com/foomo/ContentServer/server/requests"
 	"github.com/foomo/ContentServer/server/utils"
+	"strings"
 )
 
 type Repo struct {
@@ -17,18 +18,27 @@ type Repo struct {
 }
 
 func NewRepo(server string) *Repo {
+	log.Notice("creating new repo for " + server)
 	repo := new(Repo)
 	repo.server = server
 	return repo
 }
 
-func (repo *Repo) ResolveContent(URI string) (resolved bool, region string, language string, repoNode *content.RepoNode) {
-	if repoNode, ok := repo.URIDirectory[URI]; ok {
-		resolved = true
-		_, region, language := repoNode.GetLanguageAndRegionForURI(URI)
-		return true, region, language, repoNode
-	} else {
-		resolved = false
+func (repo *Repo) ResolveContent(URI string) (resolved bool, resolvedURI string, region string, language string, repoNode *content.RepoNode) {
+	parts := strings.Split(URI, content.PATH_SEPARATOR)
+	log.Debug("repo.ResolveContent: " + URI)
+	for i := len(parts); i > -1; i-- {
+		testURI := strings.Join(parts[0:i], content.PATH_SEPARATOR)
+		log.Debug("  testing" + testURI)
+		if repoNode, ok := repo.URIDirectory[testURI]; ok {
+			log.Debug("    => " + testURI)
+			resolved = true
+			_, region, language := repoNode.GetLanguageAndRegionForURI(testURI)
+			return true, testURI, region, language, repoNode
+		} else {
+			log.Debug("    => !" + testURI)
+			resolved = false
+		}
 	}
 	return
 }
@@ -50,40 +60,48 @@ func (repo *Repo) GetURI(region string, language string, id string) string {
 func (repo *Repo) GetNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []string, region string, language string) *content.Node {
 	node := content.NewNode()
 	node.Item = repoNode.ToItem(region, language)
-	for id, childNode := range repoNode.Nodes {
+	log.Debug("repo.GetNode: " + repoNode.Id)
+	for _, childId := range repoNode.Index {
+		childNode := repoNode.Nodes[childId]
 		if expanded && !childNode.Hidden && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
 			// || in path and in region mimes
-			node.Nodes[id] = repo.GetNode(childNode, expanded, mimeTypes, path, region, language)
+			node.Nodes[childId] = repo.GetNode(childNode, expanded, mimeTypes, path, region, language)
 		}
 	}
 	return node
 }
 
 func (repo *Repo) GetContent(r *requests.Content) *content.SiteContent {
+	log.Debug("repo.GetContent: " + r.URI)
 	c := content.NewSiteContent()
-	resolved, region, language, node := repo.ResolveContent(r.URI)
+	resolved, resolvedURI, region, language, node := repo.ResolveContent(r.URI)
 	if resolved {
+		log.Notice("200 for " + r.URI)
 		// forbidden ?!
 		c.Region = region
 		c.Language = language
 		c.Status = content.STATUS_OK
+		c.URI = resolvedURI
 		c.Item = node.ToItem(region, language)
 		c.Path = node.GetPath(region, language)
+		c.Data = node.Data
 		for treeName, treeRequest := range r.Nodes {
-			// fmt.Println("getting tree", treeName, treeRequest)
+			log.Debug("  adding tree " + treeName + " " + treeRequest.Id)
 			c.Nodes[treeName] = repo.GetNode(repo.Directory[treeRequest.Id], treeRequest.Expand, treeRequest.MimeTypes, []string{}, region, language)
 		}
 	} else {
+		log.Notice("404 for " + r.URI)
 		c.Status = content.STATUS_NOT_FOUND
 	}
 	return c
 }
 
 func (repo *Repo) builDirectory(dirNode *content.RepoNode) {
+	log.Debug("repo.buildDirectory: " + dirNode.Id)
 	repo.Directory[dirNode.Id] = dirNode
 	for _, languageURIs := range dirNode.URIs {
 		for _, URI := range languageURIs {
-			fmt.Println(URI, "=>", dirNode.Id)
+			log.Debug("  URI: " + URI + " => Id: " + dirNode.Id)
 			repo.URIDirectory[URI] = dirNode
 		}
 	}

server/requests/uri.go

@@ -0,0 +1,11 @@
+package requests
+
+type URI struct {
+	Id       string `json:"id"`
+	Region   string `json:"region"`
+	Language string `json:"language"`
+}
+
+func NewURI() *URI {
+	return new(URI)
+}

server/server.go

@@ -1,70 +1,28 @@
 package server
 
 import (
-	"fmt"
 	"github.com/foomo/ContentServer/server/repo"
-	"github.com/foomo/ContentServer/server/requests"
-	"github.com/foomo/ContentServer/server/utils"
-	"net/http"
-	"strings"
 )
 
-var i int = 0
+// our data
+
+type Stats struct {
+	requests int64
+}
+
+func NewStats() *Stats {
+	stats := new(Stats)
+	stats.requests = 0
+	return stats
+}
+
+var stats *Stats = NewStats()
 var contentRepo *repo.Repo
 
-func contentHandler(w http.ResponseWriter, r *http.Request) {
-	request := requests.NewContent()
-	utils.PopulateRequest(r, request)
-	utils.JsonResponse(w, contentRepo.GetContent(request))
+func countRequest() {
+	stats.requests++
 }
 
-func uriHandler(w http.ResponseWriter, r *http.Request) {
-	parts := strings.Split(r.RequestURI, "/")
-	if len(parts) == 5 {
-		utils.JsonResponse(w, contentRepo.GetURI(parts[2], parts[3], parts[4]))
-	} else {
-		wtfHandler(w, r)
-	}
-}
-
-func wtfHandler(w http.ResponseWriter, r *http.Request) {
-	fmt.Fprint(w, "thank you for your request, but i am totally lost with it\n", r.RequestURI, "\n")
-}
-
-func update() interface{} {
-	contentRepo.Update()
-	return contentRepo.Directory
-}
-
-func commandHandler(w http.ResponseWriter, r *http.Request) {
-	parts := strings.Split(r.RequestURI, "/")
-	switch true {
-	case true == (len(parts) > 1):
-		switch parts[2] {
-		case "update":
-			utils.JsonResponse(w, update())
-			return
-		default:
-			help := make(map[string]interface{})
-			help["input"] = parts[2]
-			help["commands"] = []string{"update", "help", "content"}
-			utils.JsonResponse(w, help)
-		}
-	default:
-		wtfHandler(w, r)
-	}
-}
-
-func Run(addr string, serverUrl string) {
-	fmt.Println("staring content server")
-	fmt.Printf("  loading content from %s\n", serverUrl)
-	contentRepo = repo.NewRepo(serverUrl)
-	contentRepo.Update()
-	fmt.Printf("    loaded %d items\n", len(contentRepo.Directory))
-	http.HandleFunc("/content", contentHandler)
-	http.HandleFunc("/uri/", uriHandler)
-	http.HandleFunc("/cmd/", commandHandler)
-	http.HandleFunc("/", wtfHandler)
-	fmt.Printf("  starting service on %s\n", addr)
-	http.ListenAndServe(addr, nil)
+func numRequests() int64 {
+	return stats.requests
 }

server/socket.go

@@ -0,0 +1,143 @@
+package server
+
+import (
+	"errors"
+	"fmt"
+	json "github.com/foomo/ContentServer/server/jjson"
+	"github.com/foomo/ContentServer/server/log"
+	"github.com/foomo/ContentServer/server/repo"
+	"github.com/foomo/ContentServer/server/requests"
+	"net"
+	"strconv"
+	"strings"
+)
+
+// there should be sth. built in ?!
+// anyway this ony concatenates two "ByteArrays"
+func concat(a []byte, b []byte) []byte {
+	newslice := make([]byte, len(a)+len(b))
+	copy(newslice, a)
+	copy(newslice[len(a):], b)
+	return newslice
+}
+
+func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, err error) {
+	countRequest()
+	var reply interface{}
+	var jsonErr error
+	log.Record(fmt.Sprintf("socket.handleSocketRequest(%d): %s %s", numRequests(), handler, string(jsonBuffer)))
+	switch handler {
+	case "getURI":
+		getURIRequest := requests.NewURI()
+		jsonErr = json.Unmarshal(jsonBuffer, &getURIRequest)
+		log.Debug("  getURIRequest: " + fmt.Sprint(getURIRequest))
+		uri := contentRepo.GetURI(getURIRequest.Region, getURIRequest.Language, getURIRequest.Id)
+		log.Debug("    resolved: " + uri)
+		reply = uri
+		break
+	case "content":
+		contentRequest := requests.NewContent()
+		jsonErr = json.Unmarshal(jsonBuffer, &contentRequest)
+		log.Debug("  contentRequest: " + fmt.Sprint(contentRequest))
+		content := contentRepo.GetContent(contentRequest)
+		reply = content
+		break
+	default:
+		err = errors.New(log.Error("  can not handle this one " + handler))
+	}
+	if err == nil {
+		if jsonErr != nil {
+			log.Error("  could not read incoming json: " + fmt.Sprint(jsonErr))
+			err = jsonErr
+		} else {
+			encodedBytes, jsonErr := json.MarshalIndent(map[string]interface{}{"reply": reply}, "", " ")
+			if jsonErr != nil {
+				err = jsonErr
+				log.Error("  could not encode reply " + fmt.Sprint(jsonErr))
+			} else {
+				replyBytes = encodedBytes
+			}
+		}
+	}
+	return replyBytes, err
+}
+
+func handleConnection(conn net.Conn) {
+	log.Debug("socket.handleConnection")
+	var headerBuffer [1]byte
+	header := ""
+	for {
+		_, readErr := conn.Read(headerBuffer[0:])
+		if readErr != nil {
+			log.Debug("  looks like the client closed the connection - this is my readError: " + fmt.Sprint(readErr))
+			return
+		}
+		// read next byte
+		current := string(headerBuffer[0:])
+		if current == "{" {
+			// json has started
+			headerParts := strings.Split(header, ":")
+			header = ""
+			requestHandler := headerParts[0]
+			jsonLength, _ := strconv.Atoi(headerParts[1])
+			log.Debug(fmt.Sprintf("  found json with %d bytes", jsonLength))
+			if jsonLength > 0 {
+				jsonBuffer := make([]byte, jsonLength)
+				jsonBuffer[0] = 123
+				_, jsonReadErr := conn.Read(jsonBuffer[1:])
+				if jsonReadErr != nil {
+					log.Error("  could not read json - giving up with this client connection" + fmt.Sprint(jsonReadErr))
+					return
+				} else {
+					log.Debug("  read json: " + string(jsonBuffer))
+				}
+				reply, handlingError := handleSocketRequest(requestHandler, jsonBuffer)
+				if handlingError != nil {
+					log.Error("socket.handleConnection: handlingError " + fmt.Sprint(handlingError))
+					return
+				} else {
+					headerBytes := []byte(strconv.Itoa(len(reply)))
+					reply = concat(headerBytes, reply)
+					log.Debug("  replying: " + string(reply))
+					_, writeError := conn.Write(reply)
+					if writeError != nil {
+						log.Error("socket.handleConnection: could not write my reply: " + fmt.Sprint(writeError))
+						return
+					} else {
+						log.Debug("  replied.")
+						return
+					}
+				}
+			} else {
+				log.Error("can not read empty json")
+				return
+			}
+		} else {
+			// adding to header byte by byte
+			header += string(headerBuffer[0:])
+		}
+	}
+}
+
+func RunSocketServer(server string, address string) {
+	log.Record("building repo with content from " + server)
+	contentRepo = repo.NewRepo(server)
+	contentRepo.Update()
+	ln, err := net.Listen("tcp", address)
+	if err != nil {
+		// failed to create socket
+		log.Error("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(err))
+	} else {
+		// there we go
+		log.Record("RunSocketServer: started to listen on " + address)
+		for {
+			conn, err := ln.Accept() // this blocks until connection or error
+			if err != nil {
+				log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err))
+				continue
+			} else {
+				go handleConnection(conn) // a goroutine handles conn so that the loop can accept other connections
+			}
+		}
+	}
+}

server/utils/utils.go

@@ -1,8 +1,9 @@
 package utils
 
 import (
-	"encoding/json"
+	//"encoding/json"
 	"fmt"
+	"github.com/foomo/ContentServer/server/jjson"
 	"io/ioutil"
 	"net/http"
 )
@@ -12,7 +13,8 @@ func JsonResponse(w http.ResponseWriter, obj interface{}) {
 }
 
 func toJson(obj interface{}) string {
-	b, err := json.MarshalIndent(obj, "", "\t")
+	//b, err := json.MarshalIndent(obj, "", "\t")
+	b, err := jjson.Marshal(obj)
 	if err != nil {
 		return ""
 	} else {
@@ -32,13 +34,12 @@ func extractJsonFromRequestFileUpload(r *http.Request) []byte {
 	return data
 }
 func extractJsonFromRequest(r *http.Request) []byte {
-
 	bytes := []byte(r.PostFormValue("request"))
 	return bytes
 }
 
 func PopulateRequest(r *http.Request, obj interface{}) {
-	json.Unmarshal(extractJsonFromRequest(r), obj)
+	jjson.Unmarshal(extractJsonFromRequest(r), obj)
 }
 
 func Get(URL string, obj interface{}) {
@@ -53,7 +54,7 @@ func Get(URL string, obj interface{}) {
 			fmt.Printf("%s", err)
 		}
 		// fmt.Printf("json string %s", string(contents))
-		jsonErr := json.Unmarshal(contents, &obj)
+		jsonErr := jjson.Unmarshal(contents, &obj)
 		if jsonErr != nil {
 			fmt.Println("wtf", jsonErr)
 		}