Bump gomatrixserverlib (#353)
* Bump gomatrixserverlib Mostly because I want to use Erik's go-faster jsoning. * Update KeyDB for new KeyFetcher API we now need to implement FetcherName. * Attempt to fix integ tests CanonicalJSON doesn't like the empty string, apparently, and anyway canonicalising it is pointless. * More integ test fixmain
Richard van der Hoff
GitHub
parent
9e352e7311
commit
0786318a04
|
|
@ -239,7 +239,7 @@ func testDownload(host, origin, mediaID string, wantedStatusCode int, serverCmdC
|
||||||
testReq := &test.Request{
|
testReq := &test.Request{
|
||||||
Req: req,
|
Req: req,
|
||||||
WantedStatusCode: wantedStatusCode,
|
WantedStatusCode: wantedStatusCode,
|
||||||
WantedBody: test.CanonicalJSONInput([]string{""})[0],
|
WantedBody: "",
|
||||||
}
|
}
|
||||||
testReq.Run(fmt.Sprintf("download mxc://%v/%v from %v", origin, mediaID, host), timeout, serverCmdChan)
|
testReq.Run(fmt.Sprintf("download mxc://%v/%v from %v", origin, mediaID, host), timeout, serverCmdChan)
|
||||||
}
|
}
|
||||||
|
|
@ -263,7 +263,7 @@ func testThumbnail(width, height int, resizeMethod, host string, serverCmdChan c
|
||||||
testReq := &test.Request{
|
testReq := &test.Request{
|
||||||
Req: req,
|
Req: req,
|
||||||
WantedStatusCode: 200,
|
WantedStatusCode: 200,
|
||||||
WantedBody: test.CanonicalJSONInput([]string{""})[0],
|
WantedBody: "",
|
||||||
}
|
}
|
||||||
testReq.Run(fmt.Sprintf("thumbnail mxc://%v/%v%v from %v", testOrigin, testMediaID, query, host), timeout, serverCmdChan)
|
testReq.Run(fmt.Sprintf("thumbnail mxc://%v/%v%v from %v", testOrigin, testMediaID, query, host), timeout, serverCmdChan)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,11 @@ func NewDatabase(dataSourceName string) (*Database, error) {
|
||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetcherName implements KeyFetcher
|
||||||
|
func (d Database) FetcherName() string {
|
||||||
|
return "KeyDatabase"
|
||||||
|
}
|
||||||
|
|
||||||
// FetchKeys implements gomatrixserverlib.KeyDatabase
|
// FetchKeys implements gomatrixserverlib.KeyDatabase
|
||||||
func (d *Database) FetchKeys(
|
func (d *Database) FetchKeys(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@
|
||||||
{
|
{
|
||||||
"importpath": "github.com/matrix-org/gomatrixserverlib",
|
"importpath": "github.com/matrix-org/gomatrixserverlib",
|
||||||
"repository": "https://github.com/matrix-org/gomatrixserverlib",
|
"repository": "https://github.com/matrix-org/gomatrixserverlib",
|
||||||
"revision": "076933f95312aae3a9476e78d6b4118e1b45d542",
|
"revision": "8540d3dfc13c797cd3200640bc06e0286ab355aa",
|
||||||
"branch": "master"
|
"branch": "master"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
@ -274,6 +274,24 @@
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
"path": "/require"
|
"path": "/require"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/tidwall/gjson",
|
||||||
|
"repository": "https://github.com/tidwall/gjson",
|
||||||
|
"revision": "67e2a63ac70d273b6bc7589f12f07180bc9fc189",
|
||||||
|
"branch": "master"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/tidwall/match",
|
||||||
|
"repository": "https://github.com/tidwall/match",
|
||||||
|
"revision": "1731857f09b1f38450e2c12409748407822dc6be",
|
||||||
|
"branch": "master"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/tidwall/sjson",
|
||||||
|
"repository": "https://github.com/tidwall/sjson",
|
||||||
|
"revision": "6a22caf2fd45d5e2119bfc3717e984f15a7eb7ee",
|
||||||
|
"branch": "master"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"importpath": "github.com/tj/go-debug",
|
"importpath": "github.com/tj/go-debug",
|
||||||
"repository": "https://github.com/tj/go-debug",
|
"repository": "https://github.com/tj/go-debug",
|
||||||
|
|
|
||||||
|
|
@ -175,7 +175,29 @@ func (fc *Client) LookupUserInfo(
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookupServerKeys lookups up the keys for a matrix server from a matrix server.
|
// GetServerKeys asks a matrix server for its signing keys and TLS cert
|
||||||
|
func (fc *Client) GetServerKeys(
|
||||||
|
ctx context.Context, matrixServer ServerName,
|
||||||
|
) (ServerKeys, error) {
|
||||||
|
url := url.URL{
|
||||||
|
Scheme: "matrix",
|
||||||
|
Host: string(matrixServer),
|
||||||
|
Path: "/_matrix/key/v2/server",
|
||||||
|
}
|
||||||
|
|
||||||
|
var body ServerKeys
|
||||||
|
req, err := http.NewRequest("GET", url.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return body, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = fc.DoRequestAndParseResponse(
|
||||||
|
ctx, req, &body,
|
||||||
|
)
|
||||||
|
return body, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// LookupServerKeys looks up the keys for a matrix server from a matrix server.
|
||||||
// The first argument is the name of the matrix server to download the keys from.
|
// The first argument is the name of the matrix server to download the keys from.
|
||||||
// The second argument is a map from (server name, key ID) pairs to timestamps.
|
// The second argument is a map from (server name, key ID) pairs to timestamps.
|
||||||
// The (server name, key ID) pair identifies the key to download.
|
// The (server name, key ID) pair identifies the key to download.
|
||||||
|
|
|
||||||
|
|
@ -16,11 +16,13 @@
|
||||||
package gomatrixserverlib
|
package gomatrixserverlib
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/tidwall/sjson"
|
||||||
"golang.org/x/crypto/ed25519"
|
"golang.org/x/crypto/ed25519"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -183,38 +185,53 @@ func (eb *EventBuilder) Build(eventID string, now time.Time, origin ServerName,
|
||||||
// It also checks the content hashes to ensure the event has not been tampered with.
|
// It also checks the content hashes to ensure the event has not been tampered with.
|
||||||
// This should be used when receiving new events from remote servers.
|
// This should be used when receiving new events from remote servers.
|
||||||
func NewEventFromUntrustedJSON(eventJSON []byte) (result Event, err error) {
|
func NewEventFromUntrustedJSON(eventJSON []byte) (result Event, err error) {
|
||||||
var event map[string]rawJSON
|
// We parse the JSON early on so that we don't have to check if the JSON
|
||||||
if err = json.Unmarshal(eventJSON, &event); err != nil {
|
// is valid
|
||||||
return
|
|
||||||
}
|
|
||||||
// Synapse removes these keys from events in case a server accidentally added them.
|
|
||||||
// https://github.com/matrix-org/synapse/blob/v0.18.5/synapse/crypto/event_signing.py#L57-L62
|
|
||||||
delete(event, "outlier")
|
|
||||||
delete(event, "destinations")
|
|
||||||
delete(event, "age_ts")
|
|
||||||
|
|
||||||
if eventJSON, err = json.Marshal(event); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = checkEventContentHash(eventJSON); err != nil {
|
|
||||||
result.redacted = true
|
|
||||||
// If the content hash doesn't match then we have to discard all non-essential fields
|
|
||||||
// because they've been tampered with.
|
|
||||||
if eventJSON, err = redactEvent(eventJSON); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if eventJSON, err = CanonicalJSON(eventJSON); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
result.eventJSON = eventJSON
|
|
||||||
if err = json.Unmarshal(eventJSON, &result.fields); err != nil {
|
if err = json.Unmarshal(eventJSON, &result.fields); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Synapse removes these keys from events in case a server accidentally added them.
|
||||||
|
// https://github.com/matrix-org/synapse/blob/v0.18.5/synapse/crypto/event_signing.py#L57-L62
|
||||||
|
for _, key := range []string{"outlier", "destinations", "age_ts"} {
|
||||||
|
if eventJSON, err = sjson.DeleteBytes(eventJSON, key); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We know the JSON must be valid here.
|
||||||
|
eventJSON = CanonicalJSONAssumeValid(eventJSON)
|
||||||
|
|
||||||
|
if err = checkEventContentHash(eventJSON); err != nil {
|
||||||
|
result.redacted = true
|
||||||
|
|
||||||
|
// If the content hash doesn't match then we have to discard all non-essential fields
|
||||||
|
// because they've been tampered with.
|
||||||
|
var redactedJSON []byte
|
||||||
|
if redactedJSON, err = redactEvent(eventJSON); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
redactedJSON = CanonicalJSONAssumeValid(redactedJSON)
|
||||||
|
|
||||||
|
// We need to ensure that `result` is the redacted event.
|
||||||
|
// If redactedJSON is the same as eventJSON then `result` is already
|
||||||
|
// correct. If not then we need to reparse.
|
||||||
|
//
|
||||||
|
// Yes, this means that for some events we parse twice (which is slow),
|
||||||
|
// but means that parsing unredacted events is fast.
|
||||||
|
if !bytes.Equal(redactedJSON, eventJSON) {
|
||||||
|
result = Event{redacted: true}
|
||||||
|
if err = json.Unmarshal(redactedJSON, &result.fields); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
eventJSON = redactedJSON
|
||||||
|
}
|
||||||
|
|
||||||
|
result.eventJSON = eventJSON
|
||||||
|
|
||||||
if err = result.CheckFields(); err != nil {
|
if err = result.CheckFields(); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
/* Copyright 2017 New Vector Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package gomatrixserverlib
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func benchmarkParse(b *testing.B, eventJSON string) {
|
||||||
|
var event Event
|
||||||
|
|
||||||
|
// run the Unparse function b.N times
|
||||||
|
for n := 0; n < b.N; n++ {
|
||||||
|
if err := json.Unmarshal([]byte(eventJSON), &event); err != nil {
|
||||||
|
b.Error("Failed to parse event")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Benchmark a more complicated event, in this case a power levels event.
|
||||||
|
|
||||||
|
func BenchmarkParseLargerEvent(b *testing.B) {
|
||||||
|
benchmarkParse(b, `{"auth_events":[["$Stdin0028C5qBjz5:localhost",{"sha256":"PvTyW+Mfb0aCajkIlBk1XlQE+1uVco3to8C2+/1J7iQ"}],["$klXtjBwwDQIGglax:localhost",{"sha256":"hLoiSkcGLZJr5wkIDA8+bujNJPsYX1SOCCXIErHEcgM"}]],"content":{"ban":50,"events":{"m.room.avatar":50,"m.room.canonical_alias":50,"m.room.history_visibility":100,"m.room.name":50,"m.room.power_levels":100},"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":{"@test:localhost":100},"users_default":0},"depth":3,"event_id":"$7gPR7SLdkfDsMvJL:localhost","hashes":{"sha256":"/kQnrzO5vhbnwyGvKso4CVMRyyryiyanq6t27mt5kSw"},"origin":"localhost","origin_server_ts":1510854446548,"prev_events":[["$klXtjBwwDQIGglax:localhost",{"sha256":"hLoiSkcGLZJr5wkIDA8+bujNJPsYX1SOCCXIErHEcgM"}]],"prev_state":[],"room_id":"!pUjJbIC8V32G0FLt:localhost","sender":"@test:localhost","signatures":{"localhost":{"ed25519:u9kP":"NOxjrcci7AIRhcTVmJ6nrsslLsaOJzB0iusDZ6cOFrv2OXkDY7mrBM3cQQS3DhGWltEtu3OC0nsvkfeYtwr9DQ"}},"state_key":"","type":"m.room.power_levels"}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lets now test parsing a smaller name event, first one that is valid, then wrong hash, and then the redacted one
|
||||||
|
|
||||||
|
func BenchmarkParseSmallerEvent(b *testing.B) {
|
||||||
|
benchmarkParse(b, `{"auth_events":[["$oXL79cT7fFxR7dPH:localhost",{"sha256":"abjkiDSg1RkuZrbj2jZoGMlQaaj1Ue3Jhi7I7NlKfXY"}],["$IVUsaSkm1LBAZYYh:localhost",{"sha256":"X7RUj46hM/8sUHNBIFkStbOauPvbDzjSdH4NibYWnko"}],["$VS2QT0EeArZYi8wf:localhost",{"sha256":"k9eM6utkCH8vhLW9/oRsH74jOBS/6RVK42iGDFbylno"}]],"content":{"name":"test3"},"depth":7,"event_id":"$yvN1b43rlmcOs5fY:localhost","hashes":{"sha256":"Oh1mwI1jEqZ3tgJ+V1Dmu5nOEGpCE4RFUqyJv2gQXKs"},"origin":"localhost","origin_server_ts":1510854416361,"prev_events":[["$FqI6TVvWpcbcnJ97:localhost",{"sha256":"upCsBqUhNUgT2/+zkzg8TbqdQpWWKQnZpGJc6KcbUC4"}]],"prev_state":[],"room_id":"!19Mp0U9hjajeIiw1:localhost","sender":"@test:localhost","signatures":{"localhost":{"ed25519:u9kP":"5IzSuRXkxvbTp0vZhhXYZeOe+619iG3AybJXr7zfNn/4vHz4TH7qSJVQXSaHHvcTcDodAKHnTG1WDulgO5okAQ"}},"state_key":"","type":"m.room.name"}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkParseSmallerEventFailedHash(b *testing.B) {
|
||||||
|
benchmarkParse(b, `{"auth_events":[["$oXL79cT7fFxR7dPH:localhost",{"sha256":"abjkiDSg1RkuZrbj2jZoGMlQaaj1Ue3Jhi7I7NlKfXY"}],["$IVUsaSkm1LBAZYYh:localhost",{"sha256":"X7RUj46hM/8sUHNBIFkStbOauPvbDzjSdH4NibYWnko"}],["$VS2QT0EeArZYi8wf:localhost",{"sha256":"k9eM6utkCH8vhLW9/oRsH74jOBS/6RVK42iGDFbylno"}]],"content":{"name":"test4"},"depth":7,"event_id":"$yvN1b43rlmcOs5fY:localhost","hashes":{"sha256":"Oh1mwI1jEqZ3tgJ+V1Dmu5nOEGpCE4RFUqyJv2gQXKs"},"origin":"localhost","origin_server_ts":1510854416361,"prev_events":[["$FqI6TVvWpcbcnJ97:localhost",{"sha256":"upCsBqUhNUgT2/+zkzg8TbqdQpWWKQnZpGJc6KcbUC4"}]],"prev_state":[],"room_id":"!19Mp0U9hjajeIiw1:localhost","sender":"@test:localhost","signatures":{"localhost":{"ed25519:u9kP":"5IzSuRXkxvbTp0vZhhXYZeOe+619iG3AybJXr7zfNn/4vHz4TH7qSJVQXSaHHvcTcDodAKHnTG1WDulgO5okAQ"}},"state_key":"","type":"m.room.name"}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkParseSmallerEventRedacted(b *testing.B) {
|
||||||
|
benchmarkParse(b, `{"event_id":"$yvN1b43rlmcOs5fY:localhost","sender":"@test:localhost","room_id":"!19Mp0U9hjajeIiw1:localhost","hashes":{"sha256":"Oh1mwI1jEqZ3tgJ+V1Dmu5nOEGpCE4RFUqyJv2gQXKs"},"signatures":{"localhost":{"ed25519:u9kP":"5IzSuRXkxvbTp0vZhhXYZeOe+619iG3AybJXr7zfNn/4vHz4TH7qSJVQXSaHHvcTcDodAKHnTG1WDulgO5okAQ"}},"content":{},"type":"m.room.name","state_key":"","depth":7,"prev_events":[["$FqI6TVvWpcbcnJ97:localhost",{"sha256":"upCsBqUhNUgT2/+zkzg8TbqdQpWWKQnZpGJc6KcbUC4"}]],"prev_state":[],"auth_events":[["$oXL79cT7fFxR7dPH:localhost",{"sha256":"abjkiDSg1RkuZrbj2jZoGMlQaaj1Ue3Jhi7I7NlKfXY"}],["$IVUsaSkm1LBAZYYh:localhost",{"sha256":"X7RUj46hM/8sUHNBIFkStbOauPvbDzjSdH4NibYWnko"}],["$VS2QT0EeArZYi8wf:localhost",{"sha256":"k9eM6utkCH8vhLW9/oRsH74jOBS/6RVK42iGDFbylno"}]],"origin":"localhost","origin_server_ts":1510854416361}`)
|
||||||
|
}
|
||||||
|
|
@ -22,6 +22,8 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/tidwall/gjson"
|
||||||
|
"github.com/tidwall/sjson"
|
||||||
"golang.org/x/crypto/ed25519"
|
"golang.org/x/crypto/ed25519"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -68,40 +70,28 @@ func addContentHashesToEvent(eventJSON []byte) ([]byte, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkEventContentHash checks if the unredacted content of the event matches the SHA-256 hash under the "hashes" key.
|
// checkEventContentHash checks if the unredacted content of the event matches the SHA-256 hash under the "hashes" key.
|
||||||
|
// Assumes that eventJSON has been canonicalised already.
|
||||||
func checkEventContentHash(eventJSON []byte) error {
|
func checkEventContentHash(eventJSON []byte) error {
|
||||||
var event map[string]rawJSON
|
var err error
|
||||||
|
|
||||||
if err := json.Unmarshal(eventJSON, &event); err != nil {
|
result := gjson.GetBytes(eventJSON, "hashes.sha256")
|
||||||
|
var hash Base64String
|
||||||
|
if err = hash.Decode(result.Str); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
hashesJSON := event["hashes"]
|
hashableEventJSON := eventJSON
|
||||||
|
|
||||||
delete(event, "signatures")
|
for _, key := range []string{"signatures", "unsigned", "hashes"} {
|
||||||
delete(event, "unsigned")
|
if hashableEventJSON, err = sjson.DeleteBytes(hashableEventJSON, key); err != nil {
|
||||||
delete(event, "hashes")
|
|
||||||
|
|
||||||
var hashes struct {
|
|
||||||
Sha256 Base64String `json:"sha256"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(hashesJSON, &hashes); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
hashableEventJSON, err := json.Marshal(event)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
hashableEventJSON, err = CanonicalJSON(hashableEventJSON)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sha256Hash := sha256.Sum256(hashableEventJSON)
|
sha256Hash := sha256.Sum256(hashableEventJSON)
|
||||||
|
|
||||||
if !bytes.Equal(sha256Hash[:], []byte(hashes.Sha256)) {
|
if !bytes.Equal(sha256Hash[:], []byte(hash)) {
|
||||||
return fmt.Errorf("Invalid Sha256 content hash: %v != %v", sha256Hash[:], []byte(hashes.Sha256))
|
return fmt.Errorf("Invalid Sha256 content hash: %v != %v", sha256Hash[:], []byte(hash))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/matrix-org/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A RespSend is the content of a response to PUT /_matrix/federation/v1/send/{txnID}/
|
// A RespSend is the content of a response to PUT /_matrix/federation/v1/send/{txnID}/
|
||||||
|
|
@ -109,6 +111,7 @@ func (r RespState) Events() ([]Event, error) {
|
||||||
|
|
||||||
// Check that a response to /state is valid.
|
// Check that a response to /state is valid.
|
||||||
func (r RespState) Check(ctx context.Context, keyRing JSONVerifier) error {
|
func (r RespState) Check(ctx context.Context, keyRing JSONVerifier) error {
|
||||||
|
logger := util.GetLogger(ctx)
|
||||||
var allEvents []Event
|
var allEvents []Event
|
||||||
for _, event := range r.AuthEvents {
|
for _, event := range r.AuthEvents {
|
||||||
if event.StateKey() == nil {
|
if event.StateKey() == nil {
|
||||||
|
|
@ -134,8 +137,9 @@ func (r RespState) Check(ctx context.Context, keyRing JSONVerifier) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the events pass signature checks.
|
// Check if the events pass signature checks.
|
||||||
|
logger.Infof("Checking event signatures for %d events of room state", len(allEvents))
|
||||||
if err := VerifyEventSignatures(ctx, allEvents, keyRing); err != nil {
|
if err := VerifyEventSignatures(ctx, allEvents, keyRing); err != nil {
|
||||||
return nil
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
eventsByID := map[string]*Event{}
|
eventsByID := map[string]*Event{}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,25 @@
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
# make the GIT_DIR and GIT_INDEX_FILE absolute, before we change dir
|
||||||
|
export GIT_DIR=$(readlink -f `git rev-parse --git-dir`)
|
||||||
|
if [ -n "${GIT_INDEX_FILE:+x}" ]; then
|
||||||
|
export GIT_INDEX_FILE=$(readlink -f "$GIT_INDEX_FILE")
|
||||||
|
fi
|
||||||
|
|
||||||
|
wd=`pwd`
|
||||||
|
|
||||||
|
# create a temp dir. The `trap` incantation will ensure that it is removed
|
||||||
|
# again when this script completes.
|
||||||
|
tmpdir=`mktemp -d`
|
||||||
|
trap 'rm -rf "$tmpdir"' EXIT
|
||||||
|
cd "$tmpdir"
|
||||||
|
|
||||||
|
# get a clean copy of the index (ie, what has been `git add`ed), so that we can
|
||||||
|
# run the checks against what we are about to commit, rather than what is in
|
||||||
|
# the working copy.
|
||||||
|
git checkout-index -a
|
||||||
|
|
||||||
echo "Installing lint search engine..."
|
echo "Installing lint search engine..."
|
||||||
go get github.com/alecthomas/gometalinter/
|
go get github.com/alecthomas/gometalinter/
|
||||||
gometalinter --config=linter.json --install --update
|
gometalinter --config=linter.json --install --update
|
||||||
|
|
|
||||||
|
|
@ -16,66 +16,73 @@
|
||||||
package gomatrixserverlib
|
package gomatrixserverlib
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"encoding/json"
|
|
||||||
"sort"
|
"sort"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/tidwall/gjson"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanonicalJSON re-encodes the JSON in a canonical encoding. The encoding is
|
// CanonicalJSON re-encodes the JSON in a canonical encoding. The encoding is
|
||||||
// the shortest possible encoding using integer values with sorted object keys.
|
// the shortest possible encoding using integer values with sorted object keys.
|
||||||
// https://matrix.org/docs/spec/server_server/unstable.html#canonical-json
|
// https://matrix.org/docs/spec/server_server/unstable.html#canonical-json
|
||||||
func CanonicalJSON(input []byte) ([]byte, error) {
|
func CanonicalJSON(input []byte) ([]byte, error) {
|
||||||
sorted, err := SortJSON(input, make([]byte, 0, len(input)))
|
if !gjson.Valid(string(input)) {
|
||||||
if err != nil {
|
return nil, errors.Errorf("invalid json")
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
return CompactJSON(sorted, make([]byte, 0, len(sorted))), nil
|
|
||||||
|
return CanonicalJSONAssumeValid(input), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CanonicalJSONAssumeValid is the same as CanonicalJSON, but assumes the
|
||||||
|
// input is valid JSON
|
||||||
|
func CanonicalJSONAssumeValid(input []byte) []byte {
|
||||||
|
input = CompactJSON(input, make([]byte, 0, len(input)))
|
||||||
|
return SortJSON(input, make([]byte, 0, len(input)))
|
||||||
}
|
}
|
||||||
|
|
||||||
// SortJSON reencodes the JSON with the object keys sorted by lexicographically
|
// SortJSON reencodes the JSON with the object keys sorted by lexicographically
|
||||||
// by codepoint. The input must be valid JSON.
|
// by codepoint. The input must be valid JSON.
|
||||||
func SortJSON(input, output []byte) ([]byte, error) {
|
func SortJSON(input, output []byte) []byte {
|
||||||
// Skip to the first character that isn't whitespace.
|
result := gjson.ParseBytes(input)
|
||||||
var decoded interface{}
|
|
||||||
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(input))
|
rawJSON := rawJSONFromResult(result, input)
|
||||||
decoder.UseNumber()
|
return sortJSONValue(result, rawJSON, output)
|
||||||
if err := decoder.Decode(&decoded); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return sortJSONValue(decoded, output)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortJSONValue(input interface{}, output []byte) ([]byte, error) {
|
// sortJSONValue takes a gjson.Result and sorts it. inputJSON must be the
|
||||||
switch value := input.(type) {
|
// raw JSON bytes that gjson.Result points to.
|
||||||
case []interface{}:
|
func sortJSONValue(input gjson.Result, inputJSON, output []byte) []byte {
|
||||||
// If the JSON is an array then we need to sort the keys of its children.
|
if input.IsArray() {
|
||||||
return sortJSONArray(value, output)
|
return sortJSONArray(input, inputJSON, output)
|
||||||
case map[string]interface{}:
|
|
||||||
// If the JSON is an object then we need to sort its keys and the keys of its children.
|
|
||||||
return sortJSONObject(value, output)
|
|
||||||
default:
|
|
||||||
// Otherwise the JSON is a value and can be encoded without any further sorting.
|
|
||||||
bytes, err := json.Marshal(value)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return append(output, bytes...), nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortJSONArray(input []interface{}, output []byte) ([]byte, error) {
|
if input.IsObject() {
|
||||||
var err error
|
return sortJSONObject(input, inputJSON, output)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If its neither an object nor an array then there is no sub structure
|
||||||
|
// to sort, so just append the raw bytes.
|
||||||
|
return append(output, inputJSON...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortJSONArray takes a gjson.Result and sorts it, assuming its an array.
|
||||||
|
// inputJSON must be the raw JSON bytes that gjson.Result points to.
|
||||||
|
func sortJSONArray(input gjson.Result, inputJSON, output []byte) []byte {
|
||||||
sep := byte('[')
|
sep := byte('[')
|
||||||
for _, value := range input {
|
|
||||||
|
// Iterate over each value in the array and sort it.
|
||||||
|
input.ForEach(func(_, value gjson.Result) bool {
|
||||||
output = append(output, sep)
|
output = append(output, sep)
|
||||||
sep = ','
|
sep = ','
|
||||||
if output, err = sortJSONValue(value, output); err != nil {
|
|
||||||
return nil, err
|
rawJSON := rawJSONFromResult(value, inputJSON)
|
||||||
}
|
output = sortJSONValue(value, rawJSON, output)
|
||||||
}
|
|
||||||
|
return true // keep iterating
|
||||||
|
})
|
||||||
|
|
||||||
if sep == '[' {
|
if sep == '[' {
|
||||||
// If sep is still '[' then the array was empty and we never wrote the
|
// If sep is still '[' then the array was empty and we never wrote the
|
||||||
// initial '[', so we write it now along with the closing ']'.
|
// initial '[', so we write it now along with the closing ']'.
|
||||||
|
|
@ -84,31 +91,49 @@ func sortJSONArray(input []interface{}, output []byte) ([]byte, error) {
|
||||||
// Otherwise we end the array by writing a single ']'
|
// Otherwise we end the array by writing a single ']'
|
||||||
output = append(output, ']')
|
output = append(output, ']')
|
||||||
}
|
}
|
||||||
return output, nil
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortJSONObject(input map[string]interface{}, output []byte) ([]byte, error) {
|
// sortJSONObject takes a gjson.Result and sorts it, assuming its an object.
|
||||||
var err error
|
// inputJSON must be the raw JSON bytes that gjson.Result points to.
|
||||||
keys := make([]string, len(input))
|
func sortJSONObject(input gjson.Result, inputJSON, output []byte) []byte {
|
||||||
var j int
|
type entry struct {
|
||||||
for key := range input {
|
key string // The parsed key string
|
||||||
keys[j] = key
|
rawKey []byte // The raw, unparsed key JSON string
|
||||||
j++
|
value gjson.Result
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
|
||||||
|
var entries []entry
|
||||||
|
|
||||||
|
// Iterate over each key/value pair and add it to a slice
|
||||||
|
// that we can sort
|
||||||
|
input.ForEach(func(key, value gjson.Result) bool {
|
||||||
|
entries = append(entries, entry{
|
||||||
|
key: key.String(),
|
||||||
|
rawKey: rawJSONFromResult(key, inputJSON),
|
||||||
|
value: value,
|
||||||
|
})
|
||||||
|
return true // keep iterating
|
||||||
|
})
|
||||||
|
|
||||||
|
// Sort the slice based on the *parsed* key
|
||||||
|
sort.Slice(entries, func(a, b int) bool {
|
||||||
|
return entries[a].key < entries[b].key
|
||||||
|
})
|
||||||
|
|
||||||
sep := byte('{')
|
sep := byte('{')
|
||||||
for _, key := range keys {
|
|
||||||
|
for _, entry := range entries {
|
||||||
output = append(output, sep)
|
output = append(output, sep)
|
||||||
sep = ','
|
sep = ','
|
||||||
var encoded []byte
|
|
||||||
if encoded, err = json.Marshal(key); err != nil {
|
// Append the raw unparsed JSON key, *not* the parsed key
|
||||||
return nil, err
|
output = append(output, entry.rawKey...)
|
||||||
}
|
|
||||||
output = append(output, encoded...)
|
|
||||||
output = append(output, ':')
|
output = append(output, ':')
|
||||||
if output, err = sortJSONValue(input[key], output); err != nil {
|
|
||||||
return nil, err
|
rawJSON := rawJSONFromResult(entry.value, inputJSON)
|
||||||
}
|
|
||||||
|
output = sortJSONValue(entry.value, rawJSON, output)
|
||||||
}
|
}
|
||||||
if sep == '{' {
|
if sep == '{' {
|
||||||
// If sep is still '{' then the object was empty and we never wrote the
|
// If sep is still '{' then the object was empty and we never wrote the
|
||||||
|
|
@ -118,7 +143,7 @@ func sortJSONObject(input map[string]interface{}, output []byte) ([]byte, error)
|
||||||
// Otherwise we end the object by writing a single '}'
|
// Otherwise we end the object by writing a single '}'
|
||||||
output = append(output, '}')
|
output = append(output, '}')
|
||||||
}
|
}
|
||||||
return output, nil
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompactJSON makes the encoded JSON as small as possible by removing
|
// CompactJSON makes the encoded JSON as small as possible by removing
|
||||||
|
|
@ -237,3 +262,19 @@ func readHexDigits(input []byte) uint32 {
|
||||||
hex |= hex >> 8
|
hex |= hex >> 8
|
||||||
return hex & 0xFFFF
|
return hex & 0xFFFF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rawJSONFromResult extracts the raw JSON bytes pointed to by result.
|
||||||
|
// input must be the json bytes that were used to generate result
|
||||||
|
func rawJSONFromResult(result gjson.Result, input []byte) (rawJSON []byte) {
|
||||||
|
// This is lifted from gjson README. Basically, result.Raw is a copy of
|
||||||
|
// the bytes we want, but its more efficient to take a slice.
|
||||||
|
// If Index is 0 then for some reason we can't extract it from the original
|
||||||
|
// JSON bytes.
|
||||||
|
if result.Index > 0 {
|
||||||
|
rawJSON = input[result.Index : result.Index+len(result.Raw)]
|
||||||
|
} else {
|
||||||
|
rawJSON = []byte(result.Raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,10 +20,8 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func testSortJSON(t *testing.T, input, want string) {
|
func testSortJSON(t *testing.T, input, want string) {
|
||||||
got, err := SortJSON([]byte(input), nil)
|
got := SortJSON([]byte(input), nil)
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
// Squash out the whitespace before comparing the JSON in case SortJSON had inserted whitespace.
|
// Squash out the whitespace before comparing the JSON in case SortJSON had inserted whitespace.
|
||||||
if string(CompactJSON(got, nil)) != want {
|
if string(CompactJSON(got, nil)) != want {
|
||||||
t.Errorf("SortJSON(%q): want %q got %q", input, want, got)
|
t.Errorf("SortJSON(%q): want %q got %q", input, want, got)
|
||||||
|
|
@ -36,6 +34,7 @@ func TestSortJSON(t *testing.T) {
|
||||||
`{"A":{"1":1,"2":2},"B":{"3":3,"4":4}}`)
|
`{"A":{"1":1,"2":2},"B":{"3":3,"4":4}}`)
|
||||||
testSortJSON(t, `[true,false,null]`, `[true,false,null]`)
|
testSortJSON(t, `[true,false,null]`, `[true,false,null]`)
|
||||||
testSortJSON(t, `[9007199254740991]`, `[9007199254740991]`)
|
testSortJSON(t, `[9007199254740991]`, `[9007199254740991]`)
|
||||||
|
testSortJSON(t, "\t\n[9007199254740991]", `[9007199254740991]`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testCompactJSON(t *testing.T, input, want string) {
|
func testCompactJSON(t *testing.T, input, want string) {
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/matrix-org/util"
|
||||||
"golang.org/x/crypto/ed25519"
|
"golang.org/x/crypto/ed25519"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -60,6 +61,10 @@ type KeyFetcher interface {
|
||||||
// The result may have more (server name, key ID) pairs than were in the request.
|
// The result may have more (server name, key ID) pairs than were in the request.
|
||||||
// Returns an error if there was a problem fetching the keys.
|
// Returns an error if there was a problem fetching the keys.
|
||||||
FetchKeys(ctx context.Context, requests map[PublicKeyRequest]Timestamp) (map[PublicKeyRequest]PublicKeyLookupResult, error)
|
FetchKeys(ctx context.Context, requests map[PublicKeyRequest]Timestamp) (map[PublicKeyRequest]PublicKeyLookupResult, error)
|
||||||
|
|
||||||
|
// FetcherName returns the name of this fetcher, which can then be used for
|
||||||
|
// logging errors etc.
|
||||||
|
FetcherName() string
|
||||||
}
|
}
|
||||||
|
|
||||||
// A KeyDatabase is a store for caching public keys.
|
// A KeyDatabase is a store for caching public keys.
|
||||||
|
|
@ -113,6 +118,7 @@ type JSONVerifier interface {
|
||||||
|
|
||||||
// VerifyJSONs implements JSONVerifier.
|
// VerifyJSONs implements JSONVerifier.
|
||||||
func (k KeyRing) VerifyJSONs(ctx context.Context, requests []VerifyJSONRequest) ([]VerifyJSONResult, error) { // nolint: gocyclo
|
func (k KeyRing) VerifyJSONs(ctx context.Context, requests []VerifyJSONRequest) ([]VerifyJSONResult, error) { // nolint: gocyclo
|
||||||
|
logger := util.GetLogger(ctx)
|
||||||
results := make([]VerifyJSONResult, len(requests))
|
results := make([]VerifyJSONResult, len(requests))
|
||||||
keyIDs := make([][]KeyID, len(requests))
|
keyIDs := make([][]KeyID, len(requests))
|
||||||
|
|
||||||
|
|
@ -154,7 +160,7 @@ func (k KeyRing) VerifyJSONs(ctx context.Context, requests []VerifyJSONRequest)
|
||||||
}
|
}
|
||||||
k.checkUsingKeys(requests, results, keyIDs, keysFromDatabase)
|
k.checkUsingKeys(requests, results, keyIDs, keysFromDatabase)
|
||||||
|
|
||||||
for i := range k.KeyFetchers {
|
for _, fetcher := range k.KeyFetchers {
|
||||||
// TODO: we should distinguish here between expired keys, and those we don't have.
|
// TODO: we should distinguish here between expired keys, and those we don't have.
|
||||||
// If the key has expired, it's no use re-requesting it.
|
// If the key has expired, it's no use re-requesting it.
|
||||||
keyRequests := k.publicKeyRequests(requests, results, keyIDs)
|
keyRequests := k.publicKeyRequests(requests, results, keyIDs)
|
||||||
|
|
@ -163,12 +169,22 @@ func (k KeyRing) VerifyJSONs(ctx context.Context, requests []VerifyJSONRequest)
|
||||||
// This means that we've checked every JSON object we can check.
|
// This means that we've checked every JSON object we can check.
|
||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
fetcherLogger := logger.WithField("fetcher", fetcher.FetcherName())
|
||||||
|
|
||||||
// TODO: Coalesce in-flight requests for the same keys.
|
// TODO: Coalesce in-flight requests for the same keys.
|
||||||
// Otherwise we risk spamming the servers we query the keys from.
|
// Otherwise we risk spamming the servers we query the keys from.
|
||||||
keysFetched, err := k.KeyFetchers[i].FetchKeys(ctx, keyRequests)
|
|
||||||
|
fetcherLogger.WithField("num_key_requests", len(keyRequests)).
|
||||||
|
Info("Requesting keys from fetcher")
|
||||||
|
|
||||||
|
keysFetched, err := fetcher.FetchKeys(ctx, keyRequests)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fetcherLogger.WithField("num_keys_fetched", len(keysFetched)).
|
||||||
|
Info("Got keys from fetcher")
|
||||||
|
|
||||||
k.checkUsingKeys(requests, results, keyIDs, keysFetched)
|
k.checkUsingKeys(requests, results, keyIDs, keysFetched)
|
||||||
|
|
||||||
// Add the keys to the database so that we won't need to fetch them again.
|
// Add the keys to the database so that we won't need to fetch them again.
|
||||||
|
|
@ -259,6 +275,11 @@ type PerspectiveKeyFetcher struct {
|
||||||
Client Client
|
Client Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetcherName implements KeyFetcher
|
||||||
|
func (p PerspectiveKeyFetcher) FetcherName() string {
|
||||||
|
return fmt.Sprintf("perspective server %s", p.PerspectiveServerName)
|
||||||
|
}
|
||||||
|
|
||||||
// FetchKeys implements KeyFetcher
|
// FetchKeys implements KeyFetcher
|
||||||
func (p *PerspectiveKeyFetcher) FetchKeys(
|
func (p *PerspectiveKeyFetcher) FetchKeys(
|
||||||
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
||||||
|
|
@ -303,7 +324,8 @@ func (p *PerspectiveKeyFetcher) FetchKeys(
|
||||||
return nil, fmt.Errorf("gomatrixserverlib: key response from perspective server failed checks")
|
return nil, fmt.Errorf("gomatrixserverlib: key response from perspective server failed checks")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: What happens if the same key ID appears in multiple responses?
|
// TODO (matrix-org/dendrite#345): What happens if the same key ID
|
||||||
|
// appears in multiple responses?
|
||||||
// We should probably take the response with the highest valid_until_ts.
|
// We should probably take the response with the highest valid_until_ts.
|
||||||
mapServerKeysToPublicKeyLookupResult(keys, results)
|
mapServerKeysToPublicKeyLookupResult(keys, results)
|
||||||
}
|
}
|
||||||
|
|
@ -318,6 +340,11 @@ type DirectKeyFetcher struct {
|
||||||
Client Client
|
Client Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetcherName implements KeyFetcher
|
||||||
|
func (d DirectKeyFetcher) FetcherName() string {
|
||||||
|
return "DirectKeyFetcher"
|
||||||
|
}
|
||||||
|
|
||||||
// FetchKeys implements KeyFetcher
|
// FetchKeys implements KeyFetcher
|
||||||
func (d *DirectKeyFetcher) FetchKeys(
|
func (d *DirectKeyFetcher) FetchKeys(
|
||||||
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
||||||
|
|
@ -333,9 +360,9 @@ func (d *DirectKeyFetcher) FetchKeys(
|
||||||
}
|
}
|
||||||
|
|
||||||
results := map[PublicKeyRequest]PublicKeyLookupResult{}
|
results := map[PublicKeyRequest]PublicKeyLookupResult{}
|
||||||
for server, reqs := range byServer {
|
for server := range byServer {
|
||||||
// TODO: make these requests in parallel
|
// TODO: make these requests in parallel
|
||||||
serverResults, err := d.fetchKeysForServer(ctx, server, reqs)
|
serverResults, err := d.fetchKeysForServer(ctx, server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// TODO: Should we actually be erroring here? or should we just drop those keys from the result map?
|
// TODO: Should we actually be erroring here? or should we just drop those keys from the result map?
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -348,25 +375,23 @@ func (d *DirectKeyFetcher) FetchKeys(
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DirectKeyFetcher) fetchKeysForServer(
|
func (d *DirectKeyFetcher) fetchKeysForServer(
|
||||||
ctx context.Context, serverName ServerName, requests map[PublicKeyRequest]Timestamp,
|
ctx context.Context, serverName ServerName,
|
||||||
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
||||||
serverKeys, err := d.Client.LookupServerKeys(ctx, serverName, requests)
|
keys, err := d.Client.GetServerKeys(ctx, serverName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
results := map[PublicKeyRequest]PublicKeyLookupResult{}
|
|
||||||
for _, keys := range serverKeys {
|
|
||||||
// Check that the keys are valid for the server.
|
// Check that the keys are valid for the server.
|
||||||
checks, _, _ := CheckKeys(serverName, time.Unix(0, 0), keys, nil)
|
checks, _, _ := CheckKeys(serverName, time.Unix(0, 0), keys, nil)
|
||||||
if !checks.AllChecksOK {
|
if !checks.AllChecksOK {
|
||||||
return nil, fmt.Errorf("gomatrixserverlib: key response direct from %q failed checks", serverName)
|
return nil, fmt.Errorf("gomatrixserverlib: key response direct from %q failed checks", serverName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: What happens if the same key ID appears in multiple responses?
|
results := map[PublicKeyRequest]PublicKeyLookupResult{}
|
||||||
// We should probably take the response with the highest valid_until_ts.
|
|
||||||
|
// TODO (matrix-org/dendrite#345): What happens if the same key ID
|
||||||
|
// appears in multiple responses? We should probably reject the response.
|
||||||
mapServerKeysToPublicKeyLookupResult(keys, results)
|
mapServerKeysToPublicKeyLookupResult(keys, results)
|
||||||
}
|
|
||||||
|
|
||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,10 @@ var testKeys = `{
|
||||||
|
|
||||||
type testKeyDatabase struct{}
|
type testKeyDatabase struct{}
|
||||||
|
|
||||||
|
func (db testKeyDatabase) FetcherName() string {
|
||||||
|
return "testKeyDatabase"
|
||||||
|
}
|
||||||
|
|
||||||
func (db *testKeyDatabase) FetchKeys(
|
func (db *testKeyDatabase) FetchKeys(
|
||||||
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
||||||
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
||||||
|
|
@ -151,6 +155,11 @@ func (e *erroringKeyDatabaseError) Error() string { return "An error with the ke
|
||||||
var testErrorFetch = erroringKeyDatabaseError(1)
|
var testErrorFetch = erroringKeyDatabaseError(1)
|
||||||
var testErrorStore = erroringKeyDatabaseError(2)
|
var testErrorStore = erroringKeyDatabaseError(2)
|
||||||
|
|
||||||
|
// FetcherName implements KeyFetcher
|
||||||
|
func (e erroringKeyDatabase) FetcherName() string {
|
||||||
|
return "ErroringKeyDatabase"
|
||||||
|
}
|
||||||
|
|
||||||
func (e *erroringKeyDatabase) FetchKeys(
|
func (e *erroringKeyDatabase) FetchKeys(
|
||||||
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
ctx context.Context, requests map[PublicKeyRequest]Timestamp,
|
||||||
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
) (map[PublicKeyRequest]PublicKeyLookupResult, error) {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2016 Josh Baker
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
@ -0,0 +1,373 @@
|
||||||
|
<p align="center">
|
||||||
|
<img
|
||||||
|
src="logo.png"
|
||||||
|
width="240" height="78" border="0" alt="GJSON">
|
||||||
|
<br>
|
||||||
|
<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
|
||||||
|
<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||||
|
<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/play-ground-orange.svg?style=flat-square" alt="GJSON Playground"></a>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<p align="center">get a json value quickly</a></p>
|
||||||
|
|
||||||
|
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
|
||||||
|
It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array).
|
||||||
|
|
||||||
|
Getting Started
|
||||||
|
===============
|
||||||
|
|
||||||
|
## Installing
|
||||||
|
|
||||||
|
To start using GJSON, install Go and run `go get`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ go get -u github.com/tidwall/gjson
|
||||||
|
```
|
||||||
|
|
||||||
|
This will retrieve the library.
|
||||||
|
|
||||||
|
## Get a value
|
||||||
|
Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results. When the value is found it's returned immediately.
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "github.com/tidwall/gjson"
|
||||||
|
|
||||||
|
const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
value := gjson.Get(json, "name.last")
|
||||||
|
println(value.String())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This will print:
|
||||||
|
|
||||||
|
```
|
||||||
|
Prichard
|
||||||
|
```
|
||||||
|
*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.*
|
||||||
|
|
||||||
|
## Path Syntax
|
||||||
|
|
||||||
|
A path is a series of keys separated by a dot.
|
||||||
|
A key may contain special wildcard characters '\*' and '?'.
|
||||||
|
To access an array value use the index as the key.
|
||||||
|
To get the number of elements in an array or to access a child path, use the '#' character.
|
||||||
|
The dot and wildcard characters can be escaped with '\\'.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": {"first": "Tom", "last": "Anderson"},
|
||||||
|
"age":37,
|
||||||
|
"children": ["Sara","Alex","Jack"],
|
||||||
|
"fav.movie": "Deer Hunter",
|
||||||
|
"friends": [
|
||||||
|
{"first": "Dale", "last": "Murphy", "age": 44},
|
||||||
|
{"first": "Roger", "last": "Craig", "age": 68},
|
||||||
|
{"first": "Jane", "last": "Murphy", "age": 47}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
```
|
||||||
|
"name.last" >> "Anderson"
|
||||||
|
"age" >> 37
|
||||||
|
"children" >> ["Sara","Alex","Jack"]
|
||||||
|
"children.#" >> 3
|
||||||
|
"children.1" >> "Alex"
|
||||||
|
"child*.2" >> "Jack"
|
||||||
|
"c?ildren.0" >> "Sara"
|
||||||
|
"fav\.movie" >> "Deer Hunter"
|
||||||
|
"friends.#.first" >> ["Dale","Roger","Jane"]
|
||||||
|
"friends.1.last" >> "Craig"
|
||||||
|
```
|
||||||
|
|
||||||
|
You can also query an array for the first match by using `#[...]`, or find all matches with `#[...]#`.
|
||||||
|
Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` operator.
|
||||||
|
|
||||||
|
```
|
||||||
|
friends.#[last=="Murphy"].first >> "Dale"
|
||||||
|
friends.#[last=="Murphy"]#.first >> ["Dale","Jane"]
|
||||||
|
friends.#[age>45]#.last >> ["Craig","Murphy"]
|
||||||
|
friends.#[first%"D*"].last >> "Murphy"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Result Type
|
||||||
|
|
||||||
|
GJSON supports the json types `string`, `number`, `bool`, and `null`.
|
||||||
|
Arrays and Objects are returned as their raw json types.
|
||||||
|
|
||||||
|
The `Result` type holds one of these:
|
||||||
|
|
||||||
|
```
|
||||||
|
bool, for JSON booleans
|
||||||
|
float64, for JSON numbers
|
||||||
|
string, for JSON string literals
|
||||||
|
nil, for JSON null
|
||||||
|
```
|
||||||
|
|
||||||
|
To directly access the value:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result.Type // can be String, Number, True, False, Null, or JSON
|
||||||
|
result.Str // holds the string
|
||||||
|
result.Num // holds the float64 number
|
||||||
|
result.Raw // holds the raw json
|
||||||
|
result.Index // index of raw value in original json, zero means index unknown
|
||||||
|
```
|
||||||
|
|
||||||
|
There are a variety of handy functions that work on a result:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result.Exists() bool
|
||||||
|
result.Value() interface{}
|
||||||
|
result.Int() int64
|
||||||
|
result.Uint() uint64
|
||||||
|
result.Float() float64
|
||||||
|
result.String() string
|
||||||
|
result.Bool() bool
|
||||||
|
result.Time() time.Time
|
||||||
|
result.Array() []gjson.Result
|
||||||
|
result.Map() map[string]gjson.Result
|
||||||
|
result.Get(path string) Result
|
||||||
|
result.ForEach(iterator func(key, value Result) bool)
|
||||||
|
result.Less(token Result, caseSensitive bool) bool
|
||||||
|
```
|
||||||
|
|
||||||
|
The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:
|
||||||
|
|
||||||
|
The `result.Array()` function returns back an array of values.
|
||||||
|
If the result represents a non-existent value, then an empty array will be returned.
|
||||||
|
If the result is not a JSON array, the return value will be an array containing one result.
|
||||||
|
|
||||||
|
```go
|
||||||
|
boolean >> bool
|
||||||
|
number >> float64
|
||||||
|
string >> string
|
||||||
|
null >> nil
|
||||||
|
array >> []interface{}
|
||||||
|
object >> map[string]interface{}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Get nested array values
|
||||||
|
|
||||||
|
Suppose you want all the last names from the following json:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"programmers": [
|
||||||
|
{
|
||||||
|
"firstName": "Janet",
|
||||||
|
"lastName": "McLaughlin",
|
||||||
|
}, {
|
||||||
|
"firstName": "Elliotte",
|
||||||
|
"lastName": "Hunter",
|
||||||
|
}, {
|
||||||
|
"firstName": "Jason",
|
||||||
|
"lastName": "Harold",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
You would use the path "programmers.#.lastName" like such:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result := gjson.Get(json, "programmers.#.lastName")
|
||||||
|
for _, name := range result.Array() {
|
||||||
|
println(name.String())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
You can also query an object inside an array:
|
||||||
|
|
||||||
|
```go
|
||||||
|
name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`)
|
||||||
|
println(name.String()) // prints "Elliotte"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Iterate through an object or array
|
||||||
|
|
||||||
|
The `ForEach` function allows for quickly iterating through an object or array.
|
||||||
|
The key and value are passed to the iterator function for objects.
|
||||||
|
Only the value is passed for arrays.
|
||||||
|
Returning `false` from an iterator will stop iteration.
|
||||||
|
|
||||||
|
```go
|
||||||
|
result := gjson.Get(json, "programmers")
|
||||||
|
result.ForEach(func(key, value gjson.Result) bool {
|
||||||
|
println(value.String())
|
||||||
|
return true // keep iterating
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
## Simple Parse and Get
|
||||||
|
|
||||||
|
There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result.
|
||||||
|
|
||||||
|
For example, all of these will return the same result:
|
||||||
|
|
||||||
|
```go
|
||||||
|
gjson.Parse(json).Get("name").Get("last")
|
||||||
|
gjson.Get(json, "name").Get("last")
|
||||||
|
gjson.Get(json, "name.last")
|
||||||
|
```
|
||||||
|
|
||||||
|
## Check for the existence of a value
|
||||||
|
|
||||||
|
Sometimes you just want to know if a value exists.
|
||||||
|
|
||||||
|
```go
|
||||||
|
value := gjson.Get(json, "name.last")
|
||||||
|
if !value.Exists() {
|
||||||
|
println("no last name")
|
||||||
|
} else {
|
||||||
|
println(value.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Or as one step
|
||||||
|
if gjson.Get(json, "name.last").Exists() {
|
||||||
|
println("has a last name")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Unmarshal to a map
|
||||||
|
|
||||||
|
To unmarshal to a `map[string]interface{}`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
m, ok := gjson.Parse(json).Value().(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
// not a map
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Working with Bytes
|
||||||
|
|
||||||
|
If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`.
|
||||||
|
|
||||||
|
```go
|
||||||
|
var json []byte = ...
|
||||||
|
result := gjson.GetBytes(json, path)
|
||||||
|
```
|
||||||
|
|
||||||
|
If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var json []byte = ...
|
||||||
|
result := gjson.GetBytes(json, path)
|
||||||
|
var raw []byte
|
||||||
|
if result.Index > 0 {
|
||||||
|
raw = json[result.Index:result.Index+len(result.Raw)]
|
||||||
|
} else {
|
||||||
|
raw = []byte(result.Raw)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`.
|
||||||
|
|
||||||
|
## Get multiple values at once
|
||||||
|
|
||||||
|
The `GetMany` function can be used to get multiple values at the same time, and is optimized to scan over a JSON payload once.
|
||||||
|
|
||||||
|
```go
|
||||||
|
results := gjson.GetMany(json, "name.first", "name.last", "age")
|
||||||
|
```
|
||||||
|
|
||||||
|
The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths.
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/),
|
||||||
|
[ffjson](https://github.com/pquerna/ffjson),
|
||||||
|
[EasyJSON](https://github.com/mailru/easyjson),
|
||||||
|
[jsonparser](https://github.com/buger/jsonparser),
|
||||||
|
and [json-iterator](https://github.com/json-iterator/go)
|
||||||
|
|
||||||
|
```
|
||||||
|
BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op
|
||||||
|
BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op
|
||||||
|
BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op
|
||||||
|
BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op
|
||||||
|
BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op
|
||||||
|
BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op
|
||||||
|
BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op
|
||||||
|
```
|
||||||
|
|
||||||
|
Benchmarks for the `GetMany` function:
|
||||||
|
|
||||||
|
```
|
||||||
|
BenchmarkGJSONGetMany4Paths-8 4000000 303 ns/op 112 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONGetMany8Paths-8 8000000 208 ns/op 56 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONGetMany16Paths-8 16000000 156 ns/op 56 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONGetMany32Paths-8 32000000 127 ns/op 64 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONGetMany64Paths-8 64000000 117 ns/op 64 B/op 0 allocs/op
|
||||||
|
BenchmarkGJSONGetMany128Paths-8 128000000 109 ns/op 64 B/op 0 allocs/op
|
||||||
|
```
|
||||||
|
|
||||||
|
JSON document used:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"widget": {
|
||||||
|
"debug": "on",
|
||||||
|
"window": {
|
||||||
|
"title": "Sample Konfabulator Widget",
|
||||||
|
"name": "main_window",
|
||||||
|
"width": 500,
|
||||||
|
"height": 500
|
||||||
|
},
|
||||||
|
"image": {
|
||||||
|
"src": "Images/Sun.png",
|
||||||
|
"hOffset": 250,
|
||||||
|
"vOffset": 250,
|
||||||
|
"alignment": "center"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"data": "Click Here",
|
||||||
|
"size": 36,
|
||||||
|
"style": "bold",
|
||||||
|
"vOffset": 100,
|
||||||
|
"alignment": "center",
|
||||||
|
"onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Each operation was rotated though one of the following search paths:
|
||||||
|
|
||||||
|
```
|
||||||
|
widget.window.name
|
||||||
|
widget.image.hOffset
|
||||||
|
widget.text.onMouseUp
|
||||||
|
```
|
||||||
|
|
||||||
|
For the `GetMany` benchmarks these paths are used:
|
||||||
|
|
||||||
|
```
|
||||||
|
widget.window.name
|
||||||
|
widget.image.hOffset
|
||||||
|
widget.text.onMouseUp
|
||||||
|
widget.window.title
|
||||||
|
widget.image.alignment
|
||||||
|
widget.text.style
|
||||||
|
widget.window.height
|
||||||
|
widget.image.src
|
||||||
|
widget.text.data
|
||||||
|
widget.text.size
|
||||||
|
```
|
||||||
|
|
||||||
|
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
|
||||||
|
|
||||||
|
|
||||||
|
## Contact
|
||||||
|
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
GJSON source code is available under the MIT [License](/LICENSE).
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
|
|
@ -0,0 +1,20 @@
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2016 Josh Baker
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
Match
|
||||||
|
=====
|
||||||
|
<a href="https://travis-ci.org/tidwall/match"><img src="https://img.shields.io/travis/tidwall/match.svg?style=flat-square" alt="Build Status"></a>
|
||||||
|
<a href="https://godoc.org/github.com/tidwall/match"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||||
|
|
||||||
|
Match is a very simple pattern matcher where '*' matches on any
|
||||||
|
number characters and '?' matches on any one character.
|
||||||
|
|
||||||
|
Installing
|
||||||
|
----------
|
||||||
|
|
||||||
|
```
|
||||||
|
go get -u github.com/tidwall/match
|
||||||
|
```
|
||||||
|
|
||||||
|
Example
|
||||||
|
-------
|
||||||
|
|
||||||
|
```go
|
||||||
|
match.Match("hello", "*llo")
|
||||||
|
match.Match("jello", "?ello")
|
||||||
|
match.Match("hello", "h*o")
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Contact
|
||||||
|
-------
|
||||||
|
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||||
|
|
||||||
|
License
|
||||||
|
-------
|
||||||
|
Redcon source code is available under the MIT [License](/LICENSE).
|
||||||
|
|
@ -0,0 +1,192 @@
|
||||||
|
// Match provides a simple pattern matcher with unicode support.
|
||||||
|
package match
|
||||||
|
|
||||||
|
import "unicode/utf8"
|
||||||
|
|
||||||
|
// Match returns true if str matches pattern. This is a very
|
||||||
|
// simple wildcard match where '*' matches on any number characters
|
||||||
|
// and '?' matches on any one character.
|
||||||
|
|
||||||
|
// pattern:
|
||||||
|
// { term }
|
||||||
|
// term:
|
||||||
|
// '*' matches any sequence of non-Separator characters
|
||||||
|
// '?' matches any single non-Separator character
|
||||||
|
// c matches character c (c != '*', '?', '\\')
|
||||||
|
// '\\' c matches character c
|
||||||
|
//
|
||||||
|
func Match(str, pattern string) bool {
|
||||||
|
if pattern == "*" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return deepMatch(str, pattern)
|
||||||
|
}
|
||||||
|
func deepMatch(str, pattern string) bool {
|
||||||
|
for len(pattern) > 0 {
|
||||||
|
if pattern[0] > 0x7f {
|
||||||
|
return deepMatchRune(str, pattern)
|
||||||
|
}
|
||||||
|
switch pattern[0] {
|
||||||
|
default:
|
||||||
|
if len(str) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if str[0] > 0x7f {
|
||||||
|
return deepMatchRune(str, pattern)
|
||||||
|
}
|
||||||
|
if str[0] != pattern[0] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case '?':
|
||||||
|
if len(str) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case '*':
|
||||||
|
return deepMatch(str, pattern[1:]) ||
|
||||||
|
(len(str) > 0 && deepMatch(str[1:], pattern))
|
||||||
|
}
|
||||||
|
str = str[1:]
|
||||||
|
pattern = pattern[1:]
|
||||||
|
}
|
||||||
|
return len(str) == 0 && len(pattern) == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func deepMatchRune(str, pattern string) bool {
|
||||||
|
var sr, pr rune
|
||||||
|
var srsz, prsz int
|
||||||
|
|
||||||
|
// read the first rune ahead of time
|
||||||
|
if len(str) > 0 {
|
||||||
|
if str[0] > 0x7f {
|
||||||
|
sr, srsz = utf8.DecodeRuneInString(str)
|
||||||
|
} else {
|
||||||
|
sr, srsz = rune(str[0]), 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sr, srsz = utf8.RuneError, 0
|
||||||
|
}
|
||||||
|
if len(pattern) > 0 {
|
||||||
|
if pattern[0] > 0x7f {
|
||||||
|
pr, prsz = utf8.DecodeRuneInString(pattern)
|
||||||
|
} else {
|
||||||
|
pr, prsz = rune(pattern[0]), 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pr, prsz = utf8.RuneError, 0
|
||||||
|
}
|
||||||
|
// done reading
|
||||||
|
for pr != utf8.RuneError {
|
||||||
|
switch pr {
|
||||||
|
default:
|
||||||
|
if srsz == utf8.RuneError {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if sr != pr {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case '?':
|
||||||
|
if srsz == utf8.RuneError {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case '*':
|
||||||
|
return deepMatchRune(str, pattern[prsz:]) ||
|
||||||
|
(srsz > 0 && deepMatchRune(str[srsz:], pattern))
|
||||||
|
}
|
||||||
|
str = str[srsz:]
|
||||||
|
pattern = pattern[prsz:]
|
||||||
|
// read the next runes
|
||||||
|
if len(str) > 0 {
|
||||||
|
if str[0] > 0x7f {
|
||||||
|
sr, srsz = utf8.DecodeRuneInString(str)
|
||||||
|
} else {
|
||||||
|
sr, srsz = rune(str[0]), 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sr, srsz = utf8.RuneError, 0
|
||||||
|
}
|
||||||
|
if len(pattern) > 0 {
|
||||||
|
if pattern[0] > 0x7f {
|
||||||
|
pr, prsz = utf8.DecodeRuneInString(pattern)
|
||||||
|
} else {
|
||||||
|
pr, prsz = rune(pattern[0]), 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pr, prsz = utf8.RuneError, 0
|
||||||
|
}
|
||||||
|
// done reading
|
||||||
|
}
|
||||||
|
|
||||||
|
return srsz == 0 && prsz == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var maxRuneBytes = func() []byte {
|
||||||
|
b := make([]byte, 4)
|
||||||
|
if utf8.EncodeRune(b, '\U0010FFFF') != 4 {
|
||||||
|
panic("invalid rune encoding")
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Allowable parses the pattern and determines the minimum and maximum allowable
|
||||||
|
// values that the pattern can represent.
|
||||||
|
// When the max cannot be determined, 'true' will be returned
|
||||||
|
// for infinite.
|
||||||
|
func Allowable(pattern string) (min, max string) {
|
||||||
|
if pattern == "" || pattern[0] == '*' {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
minb := make([]byte, 0, len(pattern))
|
||||||
|
maxb := make([]byte, 0, len(pattern))
|
||||||
|
var wild bool
|
||||||
|
for i := 0; i < len(pattern); i++ {
|
||||||
|
if pattern[i] == '*' {
|
||||||
|
wild = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if pattern[i] == '?' {
|
||||||
|
minb = append(minb, 0)
|
||||||
|
maxb = append(maxb, maxRuneBytes...)
|
||||||
|
} else {
|
||||||
|
minb = append(minb, pattern[i])
|
||||||
|
maxb = append(maxb, pattern[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if wild {
|
||||||
|
r, n := utf8.DecodeLastRune(maxb)
|
||||||
|
if r != utf8.RuneError {
|
||||||
|
if r < utf8.MaxRune {
|
||||||
|
r++
|
||||||
|
if r > 0x7f {
|
||||||
|
b := make([]byte, 4)
|
||||||
|
nn := utf8.EncodeRune(b, r)
|
||||||
|
maxb = append(maxb[:len(maxb)-n], b[:nn]...)
|
||||||
|
} else {
|
||||||
|
maxb = append(maxb[:len(maxb)-n], byte(r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return string(minb), string(maxb)
|
||||||
|
/*
|
||||||
|
return
|
||||||
|
if wild {
|
||||||
|
r, n := utf8.DecodeLastRune(maxb)
|
||||||
|
if r != utf8.RuneError {
|
||||||
|
if r < utf8.MaxRune {
|
||||||
|
infinite = true
|
||||||
|
} else {
|
||||||
|
r++
|
||||||
|
if r > 0x7f {
|
||||||
|
b := make([]byte, 4)
|
||||||
|
nn := utf8.EncodeRune(b, r)
|
||||||
|
maxb = append(maxb[:len(maxb)-n], b[:nn]...)
|
||||||
|
} else {
|
||||||
|
maxb = append(maxb[:len(maxb)-n], byte(r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return string(minb), string(maxb), infinite
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,408 @@
|
||||||
|
package match
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMatch(t *testing.T) {
|
||||||
|
if !Match("hello world", "hello world") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if Match("hello world", "jello world") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("hello world", "hello*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if Match("hello world", "jello*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("hello world", "hello?world") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if Match("hello world", "jello?world") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("hello world", "he*o?world") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("hello world", "he*o?wor*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("hello world", "he*o?*r*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("的情况下解析一个", "*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("的情况下解析一个", "*况下*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("的情况下解析一个", "*况?*") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
if !Match("的情况下解析一个", "的情况?解析一个") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWildcardMatch - Tests validate the logic of wild card matching.
|
||||||
|
// `WildcardMatch` supports '*' and '?' wildcards.
|
||||||
|
// Sample usage: In resource matching for folder policy validation.
|
||||||
|
func TestWildcardMatch(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
pattern string
|
||||||
|
text string
|
||||||
|
matched bool
|
||||||
|
}{
|
||||||
|
// Test case - 1.
|
||||||
|
// Test case with pattern containing key name with a prefix. Should accept the same text without a "*".
|
||||||
|
{
|
||||||
|
pattern: "my-folder/oo*",
|
||||||
|
text: "my-folder/oo",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 2.
|
||||||
|
// Test case with "*" at the end of the pattern.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*",
|
||||||
|
text: "my-folder/India/Karnataka/",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 3.
|
||||||
|
// Test case with prefixes shuffled.
|
||||||
|
// This should fail.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*",
|
||||||
|
text: "my-folder/Karnataka/India/",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 4.
|
||||||
|
// Test case with text expanded to the wildcards in the pattern.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban",
|
||||||
|
text: "my-folder/India/Karnataka/Ban",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 5.
|
||||||
|
// Test case with the keyname part is repeated as prefix several times.
|
||||||
|
// This is valid.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban",
|
||||||
|
text: "my-folder/India/Karnataka/Ban/Ban/Ban/Ban/Ban",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 6.
|
||||||
|
// Test case to validate that `*` can be expanded into multiple prefixes.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban",
|
||||||
|
text: "my-folder/India/Karnataka/Area1/Area2/Area3/Ban",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 7.
|
||||||
|
// Test case to validate that `*` can be expanded into multiple prefixes.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban",
|
||||||
|
text: "my-folder/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 8.
|
||||||
|
// Test case where the keyname part of the pattern is expanded in the text.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban",
|
||||||
|
text: "my-folder/India/Karnataka/Bangalore",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 9.
|
||||||
|
// Test case with prefixes and wildcard expanded for all "*".
|
||||||
|
{
|
||||||
|
pattern: "my-folder/In*/Ka*/Ban*",
|
||||||
|
text: "my-folder/India/Karnataka/Bangalore",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 10.
|
||||||
|
// Test case with keyname part being a wildcard in the pattern.
|
||||||
|
{pattern: "my-folder/*",
|
||||||
|
text: "my-folder/India",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 11.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/oo*",
|
||||||
|
text: "my-folder/odo",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
|
||||||
|
// Test case with pattern containing wildcard '?'.
|
||||||
|
// Test case - 12.
|
||||||
|
// "my-folder?/" matches "my-folder1/", "my-folder2/", "my-folder3" etc...
|
||||||
|
// doesn't match "myfolder/".
|
||||||
|
{
|
||||||
|
pattern: "my-folder?/abc*",
|
||||||
|
text: "myfolder/abc",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 13.
|
||||||
|
{
|
||||||
|
pattern: "my-folder?/abc*",
|
||||||
|
text: "my-folder1/abc",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 14.
|
||||||
|
{
|
||||||
|
pattern: "my-?-folder/abc*",
|
||||||
|
text: "my--folder/abc",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 15.
|
||||||
|
{
|
||||||
|
pattern: "my-?-folder/abc*",
|
||||||
|
text: "my-1-folder/abc",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 16.
|
||||||
|
{
|
||||||
|
pattern: "my-?-folder/abc*",
|
||||||
|
text: "my-k-folder/abc",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 17.
|
||||||
|
{
|
||||||
|
pattern: "my??folder/abc*",
|
||||||
|
text: "myfolder/abc",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 18.
|
||||||
|
{
|
||||||
|
pattern: "my??folder/abc*",
|
||||||
|
text: "my4afolder/abc",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 19.
|
||||||
|
{
|
||||||
|
pattern: "my-folder?abc*",
|
||||||
|
text: "my-folder/abc",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 20-21.
|
||||||
|
// '?' matches '/' too. (works with s3).
|
||||||
|
// This is because the namespace is considered flat.
|
||||||
|
// "abc?efg" matches both "abcdefg" and "abc/efg".
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc?efg",
|
||||||
|
text: "my-folder/abcdefg",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc?efg",
|
||||||
|
text: "my-folder/abc/efg",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case - 22.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc????",
|
||||||
|
text: "my-folder/abc",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 23.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc????",
|
||||||
|
text: "my-folder/abcde",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case - 24.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc????",
|
||||||
|
text: "my-folder/abcdefg",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 25-26.
|
||||||
|
// test case with no '*'.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc?",
|
||||||
|
text: "my-folder/abc",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc?",
|
||||||
|
text: "my-folder/abcd",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
pattern: "my-folder/abc?",
|
||||||
|
text: "my-folder/abcde",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 27.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnop",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 28.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnopqrst/mnopqr",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 29.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnopqrst/mnopqrs",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 30.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnop",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 31.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnopq",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 32.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnopqr",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 33.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopqand",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 34.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopand",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 35.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopqand",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 36.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mn",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 37.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?",
|
||||||
|
text: "my-folder/mnopqrst/mnopqrs",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 38.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*??",
|
||||||
|
text: "my-folder/mnopqrst",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 39.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*qrst",
|
||||||
|
text: "my-folder/mnopabcdegqrst",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 40.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopqand",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 41.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopand",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 42.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and?",
|
||||||
|
text: "my-folder/mnopqanda",
|
||||||
|
matched: true,
|
||||||
|
},
|
||||||
|
// Test case 43.
|
||||||
|
{
|
||||||
|
pattern: "my-folder/mnop*?and",
|
||||||
|
text: "my-folder/mnopqanda",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
// Test case 44.
|
||||||
|
|
||||||
|
{
|
||||||
|
pattern: "my-?-folder/abc*",
|
||||||
|
text: "my-folder/mnopqanda",
|
||||||
|
matched: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
// Iterating over the test cases, call the function under test and asert the output.
|
||||||
|
for i, testCase := range testCases {
|
||||||
|
actualResult := Match(testCase.text, testCase.pattern)
|
||||||
|
if testCase.matched != actualResult {
|
||||||
|
t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestRandomInput(t *testing.T) {
|
||||||
|
rand.Seed(time.Now().UnixNano())
|
||||||
|
b1 := make([]byte, 100)
|
||||||
|
b2 := make([]byte, 100)
|
||||||
|
for i := 0; i < 1000000; i++ {
|
||||||
|
if _, err := rand.Read(b1); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := rand.Read(b2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
Match(string(b1), string(b2))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func testAllowable(pattern, exmin, exmax string) error {
|
||||||
|
min, max := Allowable(pattern)
|
||||||
|
if min != exmin || max != exmax {
|
||||||
|
return fmt.Errorf("expected '%v'/'%v', got '%v'/'%v'",
|
||||||
|
exmin, exmax, min, max)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func TestAllowable(t *testing.T) {
|
||||||
|
if err := testAllowable("hell*", "hell", "helm"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := testAllowable("hell?", "hell"+string(0), "hell"+string(utf8.MaxRune)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := testAllowable("h解析ell*", "h解析ell", "h解析elm"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := testAllowable("h解*ell*", "h解", "h觤"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func BenchmarkAscii(t *testing.B) {
|
||||||
|
for i := 0; i < t.N; i++ {
|
||||||
|
if !Match("hello", "hello") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnicode(t *testing.B) {
|
||||||
|
for i := 0; i < t.N; i++ {
|
||||||
|
if !Match("h情llo", "h情llo") {
|
||||||
|
t.Fatal("fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2016 Josh Baker
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
@ -0,0 +1,278 @@
|
||||||
|
<p align="center">
|
||||||
|
<img
|
||||||
|
src="logo.png"
|
||||||
|
width="240" height="78" border="0" alt="SJSON">
|
||||||
|
<br>
|
||||||
|
<a href="https://travis-ci.org/tidwall/sjson"><img src="https://img.shields.io/travis/tidwall/sjson.svg?style=flat-square" alt="Build Status"></a>
|
||||||
|
<a href="https://godoc.org/github.com/tidwall/sjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<p align="center">set a json value quickly</a></p>
|
||||||
|
|
||||||
|
SJSON is a Go package that provides a [very fast](#performance) and simple way to set a value in a json document. The purpose for this library is to provide efficient json updating for the [SummitDB](https://github.com/tidwall/summitdb) project.
|
||||||
|
For quickly retrieving json values check out [GJSON](https://github.com/tidwall/gjson).
|
||||||
|
|
||||||
|
For a command line interface check out [JSONed](https://github.com/tidwall/jsoned).
|
||||||
|
|
||||||
|
Getting Started
|
||||||
|
===============
|
||||||
|
|
||||||
|
Installing
|
||||||
|
----------
|
||||||
|
|
||||||
|
To start using SJSON, install Go and run `go get`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ go get -u github.com/tidwall/sjson
|
||||||
|
```
|
||||||
|
|
||||||
|
This will retrieve the library.
|
||||||
|
|
||||||
|
Set a value
|
||||||
|
-----------
|
||||||
|
Set sets the value for the specified path.
|
||||||
|
A path is in dot syntax, such as "name.last" or "age".
|
||||||
|
This function expects that the json is well-formed and validated.
|
||||||
|
Invalid json will not panic, but it may return back unexpected results.
|
||||||
|
Invalid paths may return an error.
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "github.com/tidwall/sjson"
|
||||||
|
|
||||||
|
const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
value, _ := sjson.Set(json, "name.last", "Anderson")
|
||||||
|
println(value)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This will print:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"name":{"first":"Janet","last":"Anderson"},"age":47}
|
||||||
|
```
|
||||||
|
|
||||||
|
Path syntax
|
||||||
|
-----------
|
||||||
|
|
||||||
|
A path is a series of keys separated by a dot.
|
||||||
|
The dot and colon characters can be escaped with '\'.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": {"first": "Tom", "last": "Anderson"},
|
||||||
|
"age":37,
|
||||||
|
"children": ["Sara","Alex","Jack"],
|
||||||
|
"fav.movie": "Deer Hunter",
|
||||||
|
"friends": [
|
||||||
|
{"first": "James", "last": "Murphy"},
|
||||||
|
{"first": "Roger", "last": "Craig"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
```
|
||||||
|
"name.last" >> "Anderson"
|
||||||
|
"age" >> 37
|
||||||
|
"children.1" >> "Alex"
|
||||||
|
"friends.1.last" >> "Craig"
|
||||||
|
```
|
||||||
|
|
||||||
|
The `-1` key can be used to append a value to an existing array:
|
||||||
|
|
||||||
|
```
|
||||||
|
"children.-1" >> appends a new value to the end of the children array
|
||||||
|
```
|
||||||
|
|
||||||
|
Normally number keys are used to modify arrays, but it's possible to force a numeric object key by using the colon character:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"users":{
|
||||||
|
"2313":{"name":"Sara"},
|
||||||
|
"7839":{"name":"Andy"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A colon path would look like:
|
||||||
|
|
||||||
|
```
|
||||||
|
"users.:2313.name" >> "Sara"
|
||||||
|
```
|
||||||
|
|
||||||
|
Supported types
|
||||||
|
---------------
|
||||||
|
|
||||||
|
Pretty much any type is supported:
|
||||||
|
|
||||||
|
```go
|
||||||
|
sjson.Set(`{"key":true}`, "key", nil)
|
||||||
|
sjson.Set(`{"key":true}`, "key", false)
|
||||||
|
sjson.Set(`{"key":true}`, "key", 1)
|
||||||
|
sjson.Set(`{"key":true}`, "key", 10.5)
|
||||||
|
sjson.Set(`{"key":true}`, "key", "hello")
|
||||||
|
sjson.Set(`{"key":true}`, "key", map[string]interface{}{"hello":"world"})
|
||||||
|
```
|
||||||
|
|
||||||
|
When a type is not recognized, SJSON will fallback to the `encoding/json` Marshaller.
|
||||||
|
|
||||||
|
|
||||||
|
Examples
|
||||||
|
--------
|
||||||
|
|
||||||
|
Set a value from empty document:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set("", "name", "Tom")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"name":"Tom"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Set a nested value from empty document:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set("", "name.last", "Anderson")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"name":{"last":"Anderson"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Set a new value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.first", "Sara")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"name":{"first":"Sara","last":"Anderson"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Update an existing value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.last", "Smith")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"name":{"last":"Smith"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Set a new array value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.2", "Sara")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"friends":["Andy","Carol","Sara"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Append an array value by using the `-1` key in a path:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.-1", "Sara")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"friends":["Andy","Carol","Sara"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Append an array value that is past the end:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.4", "Sara")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"friends":["Andy","Carol",null,null,"Sara"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete a value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Delete(`{"name":{"first":"Sara","last":"Anderson"}}`, "name.first")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"name":{"last":"Anderson"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete an array value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.1")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"friends":["Andy"]}
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the last array value:
|
||||||
|
```go
|
||||||
|
value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.-1")
|
||||||
|
println(value)
|
||||||
|
|
||||||
|
// Output:
|
||||||
|
// {"friends":["Andy"]}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
Benchmarks of SJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/),
|
||||||
|
[ffjson](https://github.com/pquerna/ffjson),
|
||||||
|
[EasyJSON](https://github.com/mailru/easyjson),
|
||||||
|
and [Gabs](https://github.com/Jeffail/gabs)
|
||||||
|
|
||||||
|
```
|
||||||
|
Benchmark_SJSON-8 3000000 805 ns/op 1077 B/op 3 allocs/op
|
||||||
|
Benchmark_SJSON_ReplaceInPlace-8 3000000 449 ns/op 0 B/op 0 allocs/op
|
||||||
|
Benchmark_JSON_Map-8 300000 21236 ns/op 6392 B/op 150 allocs/op
|
||||||
|
Benchmark_JSON_Struct-8 300000 14691 ns/op 1789 B/op 24 allocs/op
|
||||||
|
Benchmark_Gabs-8 300000 21311 ns/op 6752 B/op 150 allocs/op
|
||||||
|
Benchmark_FFJSON-8 300000 17673 ns/op 3589 B/op 47 allocs/op
|
||||||
|
Benchmark_EasyJSON-8 1500000 3119 ns/op 1061 B/op 13 allocs/op
|
||||||
|
```
|
||||||
|
|
||||||
|
JSON document used:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"widget": {
|
||||||
|
"debug": "on",
|
||||||
|
"window": {
|
||||||
|
"title": "Sample Konfabulator Widget",
|
||||||
|
"name": "main_window",
|
||||||
|
"width": 500,
|
||||||
|
"height": 500
|
||||||
|
},
|
||||||
|
"image": {
|
||||||
|
"src": "Images/Sun.png",
|
||||||
|
"hOffset": 250,
|
||||||
|
"vOffset": 250,
|
||||||
|
"alignment": "center"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"data": "Click Here",
|
||||||
|
"size": 36,
|
||||||
|
"style": "bold",
|
||||||
|
"vOffset": 100,
|
||||||
|
"alignment": "center",
|
||||||
|
"onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Each operation was rotated though one of the following search paths:
|
||||||
|
|
||||||
|
```
|
||||||
|
widget.window.name
|
||||||
|
widget.image.hOffset
|
||||||
|
widget.text.onMouseUp
|
||||||
|
```
|
||||||
|
|
||||||
|
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.*
|
||||||
|
|
||||||
|
## Contact
|
||||||
|
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
SJSON source code is available under the MIT [License](/LICENSE).
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
|
|
@ -0,0 +1,653 @@
|
||||||
|
// Package sjson provides setting json values.
|
||||||
|
package sjson
|
||||||
|
|
||||||
|
import (
|
||||||
|
jsongo "encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/tidwall/gjson"
|
||||||
|
)
|
||||||
|
|
||||||
|
type errorType struct {
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err *errorType) Error() string {
|
||||||
|
return err.msg
|
||||||
|
}
|
||||||
|
|
||||||
|
// Options represents additional options for the Set and Delete functions.
|
||||||
|
type Options struct {
|
||||||
|
// Optimistic is a hint that the value likely exists which
|
||||||
|
// allows for the sjson to perform a fast-track search and replace.
|
||||||
|
Optimistic bool
|
||||||
|
// ReplaceInPlace is a hint to replace the input json rather than
|
||||||
|
// allocate a new json byte slice. When this field is specified
|
||||||
|
// the input json will not longer be valid and it should not be used
|
||||||
|
// In the case when the destination slice doesn't have enough free
|
||||||
|
// bytes to replace the data in place, a new bytes slice will be
|
||||||
|
// created under the hood.
|
||||||
|
// The Optimistic flag must be set to true and the input must be a
|
||||||
|
// byte slice in order to use this field.
|
||||||
|
ReplaceInPlace bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type pathResult struct {
|
||||||
|
part string // current key part
|
||||||
|
path string // remaining path
|
||||||
|
force bool // force a string key
|
||||||
|
more bool // there is more path to parse
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePath(path string) (pathResult, error) {
|
||||||
|
var r pathResult
|
||||||
|
if len(path) > 0 && path[0] == ':' {
|
||||||
|
r.force = true
|
||||||
|
path = path[1:]
|
||||||
|
}
|
||||||
|
for i := 0; i < len(path); i++ {
|
||||||
|
if path[i] == '.' {
|
||||||
|
r.part = path[:i]
|
||||||
|
r.path = path[i+1:]
|
||||||
|
r.more = true
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
if path[i] == '*' || path[i] == '?' {
|
||||||
|
return r, &errorType{"wildcard characters not allowed in path"}
|
||||||
|
} else if path[i] == '#' {
|
||||||
|
return r, &errorType{"array access character not allowed in path"}
|
||||||
|
}
|
||||||
|
if path[i] == '\\' {
|
||||||
|
// go into escape mode. this is a slower path that
|
||||||
|
// strips off the escape character from the part.
|
||||||
|
epart := []byte(path[:i])
|
||||||
|
i++
|
||||||
|
if i < len(path) {
|
||||||
|
epart = append(epart, path[i])
|
||||||
|
i++
|
||||||
|
for ; i < len(path); i++ {
|
||||||
|
if path[i] == '\\' {
|
||||||
|
i++
|
||||||
|
if i < len(path) {
|
||||||
|
epart = append(epart, path[i])
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
} else if path[i] == '.' {
|
||||||
|
r.part = string(epart)
|
||||||
|
r.path = path[i+1:]
|
||||||
|
r.more = true
|
||||||
|
return r, nil
|
||||||
|
} else if path[i] == '*' || path[i] == '?' {
|
||||||
|
return r, &errorType{
|
||||||
|
"wildcard characters not allowed in path"}
|
||||||
|
} else if path[i] == '#' {
|
||||||
|
return r, &errorType{
|
||||||
|
"array access character not allowed in path"}
|
||||||
|
}
|
||||||
|
epart = append(epart, path[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// append the last part
|
||||||
|
r.part = string(epart)
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.part = path
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustMarshalString(s string) bool {
|
||||||
|
for i := 0; i < len(s); i++ {
|
||||||
|
if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendStringify makes a json string and appends to buf.
|
||||||
|
func appendStringify(buf []byte, s string) []byte {
|
||||||
|
if mustMarshalString(s) {
|
||||||
|
b, _ := jsongo.Marshal(s)
|
||||||
|
return append(buf, b...)
|
||||||
|
}
|
||||||
|
buf = append(buf, '"')
|
||||||
|
buf = append(buf, s...)
|
||||||
|
buf = append(buf, '"')
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendBuild builds a json block from a json path.
|
||||||
|
func appendBuild(buf []byte, array bool, paths []pathResult, raw string,
|
||||||
|
stringify bool) []byte {
|
||||||
|
if !array {
|
||||||
|
buf = appendStringify(buf, paths[0].part)
|
||||||
|
buf = append(buf, ':')
|
||||||
|
}
|
||||||
|
if len(paths) > 1 {
|
||||||
|
n, numeric := atoui(paths[1])
|
||||||
|
if numeric || (!paths[1].force && paths[1].part == "-1") {
|
||||||
|
buf = append(buf, '[')
|
||||||
|
buf = appendRepeat(buf, "null,", n)
|
||||||
|
buf = appendBuild(buf, true, paths[1:], raw, stringify)
|
||||||
|
buf = append(buf, ']')
|
||||||
|
} else {
|
||||||
|
buf = append(buf, '{')
|
||||||
|
buf = appendBuild(buf, false, paths[1:], raw, stringify)
|
||||||
|
buf = append(buf, '}')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if stringify {
|
||||||
|
buf = appendStringify(buf, raw)
|
||||||
|
} else {
|
||||||
|
buf = append(buf, raw...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// atoui does a rip conversion of string -> unigned int.
|
||||||
|
func atoui(r pathResult) (n int, ok bool) {
|
||||||
|
if r.force {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
for i := 0; i < len(r.part); i++ {
|
||||||
|
if r.part[i] < '0' || r.part[i] > '9' {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
n = n*10 + int(r.part[i]-'0')
|
||||||
|
}
|
||||||
|
return n, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendRepeat repeats string "n" times and appends to buf.
|
||||||
|
func appendRepeat(buf []byte, s string, n int) []byte {
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
buf = append(buf, s...)
|
||||||
|
}
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// trim does a rip trim
|
||||||
|
func trim(s string) string {
|
||||||
|
for len(s) > 0 {
|
||||||
|
if s[0] <= ' ' {
|
||||||
|
s = s[1:]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
for len(s) > 0 {
|
||||||
|
if s[len(s)-1] <= ' ' {
|
||||||
|
s = s[:len(s)-1]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteTailItem deletes the previous key or comma.
|
||||||
|
func deleteTailItem(buf []byte) ([]byte, bool) {
|
||||||
|
loop:
|
||||||
|
for i := len(buf) - 1; i >= 0; i-- {
|
||||||
|
// look for either a ',',':','['
|
||||||
|
switch buf[i] {
|
||||||
|
case '[':
|
||||||
|
return buf, true
|
||||||
|
case ',':
|
||||||
|
return buf[:i], false
|
||||||
|
case ':':
|
||||||
|
// delete tail string
|
||||||
|
i--
|
||||||
|
for ; i >= 0; i-- {
|
||||||
|
if buf[i] == '"' {
|
||||||
|
i--
|
||||||
|
for ; i >= 0; i-- {
|
||||||
|
if buf[i] == '"' {
|
||||||
|
i--
|
||||||
|
if i >= 0 && i == '\\' {
|
||||||
|
i--
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for ; i >= 0; i-- {
|
||||||
|
// look for either a ',','{'
|
||||||
|
switch buf[i] {
|
||||||
|
case '{':
|
||||||
|
return buf[:i+1], true
|
||||||
|
case ',':
|
||||||
|
return buf[:i], false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break loop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buf, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var errNoChange = &errorType{"no change"}
|
||||||
|
|
||||||
|
func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string,
|
||||||
|
stringify, del bool) ([]byte, error) {
|
||||||
|
var err error
|
||||||
|
var res gjson.Result
|
||||||
|
var found bool
|
||||||
|
if del {
|
||||||
|
if paths[0].part == "-1" && !paths[0].force {
|
||||||
|
res = gjson.Get(jstr, "#")
|
||||||
|
if res.Int() > 0 {
|
||||||
|
res = gjson.Get(jstr, strconv.FormatInt(int64(res.Int()-1), 10))
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
res = gjson.Get(jstr, paths[0].part)
|
||||||
|
}
|
||||||
|
if res.Index > 0 {
|
||||||
|
if len(paths) > 1 {
|
||||||
|
buf = append(buf, jstr[:res.Index]...)
|
||||||
|
buf, err = appendRawPaths(buf, res.Raw, paths[1:], raw,
|
||||||
|
stringify, del)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
buf = append(buf, jstr[res.Index+len(res.Raw):]...)
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
buf = append(buf, jstr[:res.Index]...)
|
||||||
|
var exidx int // additional forward stripping
|
||||||
|
if del {
|
||||||
|
var delNextComma bool
|
||||||
|
buf, delNextComma = deleteTailItem(buf)
|
||||||
|
if delNextComma {
|
||||||
|
i, j := res.Index+len(res.Raw), 0
|
||||||
|
for ; i < len(jstr); i, j = i+1, j+1 {
|
||||||
|
if jstr[i] <= ' ' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if jstr[i] == ',' {
|
||||||
|
exidx = j + 1
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if stringify {
|
||||||
|
buf = appendStringify(buf, raw)
|
||||||
|
} else {
|
||||||
|
buf = append(buf, raw...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf = append(buf, jstr[res.Index+len(res.Raw)+exidx:]...)
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
if del {
|
||||||
|
return nil, errNoChange
|
||||||
|
}
|
||||||
|
n, numeric := atoui(paths[0])
|
||||||
|
isempty := true
|
||||||
|
for i := 0; i < len(jstr); i++ {
|
||||||
|
if jstr[i] > ' ' {
|
||||||
|
isempty = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if isempty {
|
||||||
|
if numeric {
|
||||||
|
jstr = "[]"
|
||||||
|
} else {
|
||||||
|
jstr = "{}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jsres := gjson.Parse(jstr)
|
||||||
|
if jsres.Type != gjson.JSON {
|
||||||
|
if numeric {
|
||||||
|
jstr = "[]"
|
||||||
|
} else {
|
||||||
|
jstr = "{}"
|
||||||
|
}
|
||||||
|
jsres = gjson.Parse(jstr)
|
||||||
|
}
|
||||||
|
var comma bool
|
||||||
|
for i := 1; i < len(jsres.Raw); i++ {
|
||||||
|
if jsres.Raw[i] <= ' ' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if jsres.Raw[i] == '}' || jsres.Raw[i] == ']' {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
comma = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
switch jsres.Raw[0] {
|
||||||
|
default:
|
||||||
|
return nil, &errorType{"json must be an object or array"}
|
||||||
|
case '{':
|
||||||
|
buf = append(buf, '{')
|
||||||
|
buf = appendBuild(buf, false, paths, raw, stringify)
|
||||||
|
if comma {
|
||||||
|
buf = append(buf, ',')
|
||||||
|
}
|
||||||
|
buf = append(buf, jsres.Raw[1:]...)
|
||||||
|
return buf, nil
|
||||||
|
case '[':
|
||||||
|
var appendit bool
|
||||||
|
if !numeric {
|
||||||
|
if paths[0].part == "-1" && !paths[0].force {
|
||||||
|
appendit = true
|
||||||
|
} else {
|
||||||
|
return nil, &errorType{
|
||||||
|
"cannot set array element for non-numeric key '" +
|
||||||
|
paths[0].part + "'"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if appendit {
|
||||||
|
njson := trim(jsres.Raw)
|
||||||
|
if njson[len(njson)-1] == ']' {
|
||||||
|
njson = njson[:len(njson)-1]
|
||||||
|
}
|
||||||
|
buf = append(buf, njson...)
|
||||||
|
if comma {
|
||||||
|
buf = append(buf, ',')
|
||||||
|
}
|
||||||
|
|
||||||
|
buf = appendBuild(buf, true, paths, raw, stringify)
|
||||||
|
buf = append(buf, ']')
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
buf = append(buf, '[')
|
||||||
|
ress := jsres.Array()
|
||||||
|
for i := 0; i < len(ress); i++ {
|
||||||
|
if i > 0 {
|
||||||
|
buf = append(buf, ',')
|
||||||
|
}
|
||||||
|
buf = append(buf, ress[i].Raw...)
|
||||||
|
}
|
||||||
|
if len(ress) == 0 {
|
||||||
|
buf = appendRepeat(buf, "null,", n-len(ress))
|
||||||
|
} else {
|
||||||
|
buf = appendRepeat(buf, ",null", n-len(ress))
|
||||||
|
if comma {
|
||||||
|
buf = append(buf, ',')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf = appendBuild(buf, true, paths, raw, stringify)
|
||||||
|
buf = append(buf, ']')
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isOptimisticPath(path string) bool {
|
||||||
|
for i := 0; i < len(path); i++ {
|
||||||
|
if path[i] < '.' || path[i] > 'z' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if path[i] > '9' && path[i] < 'A' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if path[i] > 'z' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func set(jstr, path, raw string,
|
||||||
|
stringify, del, optimistic, inplace bool) ([]byte, error) {
|
||||||
|
if path == "" {
|
||||||
|
return nil, &errorType{"path cannot be empty"}
|
||||||
|
}
|
||||||
|
if !del && optimistic && isOptimisticPath(path) {
|
||||||
|
res := gjson.Get(jstr, path)
|
||||||
|
if res.Exists() && res.Index > 0 {
|
||||||
|
sz := len(jstr) - len(res.Raw) + len(raw)
|
||||||
|
if stringify {
|
||||||
|
sz += 2
|
||||||
|
}
|
||||||
|
if inplace && sz <= len(jstr) {
|
||||||
|
if !stringify || !mustMarshalString(raw) {
|
||||||
|
jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr))
|
||||||
|
jsonbh := reflect.SliceHeader{
|
||||||
|
Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len}
|
||||||
|
jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh))
|
||||||
|
if stringify {
|
||||||
|
jbytes[res.Index] = '"'
|
||||||
|
copy(jbytes[res.Index+1:], []byte(raw))
|
||||||
|
jbytes[res.Index+1+len(raw)] = '"'
|
||||||
|
copy(jbytes[res.Index+1+len(raw)+1:],
|
||||||
|
jbytes[res.Index+len(res.Raw):])
|
||||||
|
} else {
|
||||||
|
copy(jbytes[res.Index:], []byte(raw))
|
||||||
|
copy(jbytes[res.Index+len(raw):],
|
||||||
|
jbytes[res.Index+len(res.Raw):])
|
||||||
|
}
|
||||||
|
return jbytes[:sz], nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
buf := make([]byte, 0, sz)
|
||||||
|
buf = append(buf, jstr[:res.Index]...)
|
||||||
|
if stringify {
|
||||||
|
buf = appendStringify(buf, raw)
|
||||||
|
} else {
|
||||||
|
buf = append(buf, raw...)
|
||||||
|
}
|
||||||
|
buf = append(buf, jstr[res.Index+len(res.Raw):]...)
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// parse the path, make sure that it does not contain invalid characters
|
||||||
|
// such as '#', '?', '*'
|
||||||
|
paths := make([]pathResult, 0, 4)
|
||||||
|
r, err := parsePath(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
paths = append(paths, r)
|
||||||
|
for r.more {
|
||||||
|
if r, err = parsePath(r.path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
paths = append(paths, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return njson, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set sets a json value for the specified path.
|
||||||
|
// A path is in dot syntax, such as "name.last" or "age".
|
||||||
|
// This function expects that the json is well-formed, and does not validate.
|
||||||
|
// Invalid json will not panic, but it may return back unexpected results.
|
||||||
|
// An error is returned if the path is not valid.
|
||||||
|
//
|
||||||
|
// A path is a series of keys separated by a dot.
|
||||||
|
//
|
||||||
|
// {
|
||||||
|
// "name": {"first": "Tom", "last": "Anderson"},
|
||||||
|
// "age":37,
|
||||||
|
// "children": ["Sara","Alex","Jack"],
|
||||||
|
// "friends": [
|
||||||
|
// {"first": "James", "last": "Murphy"},
|
||||||
|
// {"first": "Roger", "last": "Craig"}
|
||||||
|
// ]
|
||||||
|
// }
|
||||||
|
// "name.last" >> "Anderson"
|
||||||
|
// "age" >> 37
|
||||||
|
// "children.1" >> "Alex"
|
||||||
|
//
|
||||||
|
func Set(json, path string, value interface{}) (string, error) {
|
||||||
|
return SetOptions(json, path, value, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetOptions sets a json value for the specified path with options.
|
||||||
|
// A path is in dot syntax, such as "name.last" or "age".
|
||||||
|
// This function expects that the json is well-formed, and does not validate.
|
||||||
|
// Invalid json will not panic, but it may return back unexpected results.
|
||||||
|
// An error is returned if the path is not valid.
|
||||||
|
func SetOptions(json, path string, value interface{},
|
||||||
|
opts *Options) (string, error) {
|
||||||
|
if opts != nil {
|
||||||
|
if opts.ReplaceInPlace {
|
||||||
|
// it's not safe to replace bytes in-place for strings
|
||||||
|
// copy the Options and set options.ReplaceInPlace to false.
|
||||||
|
nopts := *opts
|
||||||
|
opts = &nopts
|
||||||
|
opts.ReplaceInPlace = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json))
|
||||||
|
jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len}
|
||||||
|
jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh))
|
||||||
|
res, err := SetBytesOptions(jsonb, path, value, opts)
|
||||||
|
return string(res), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetBytes sets a json value for the specified path.
|
||||||
|
// If working with bytes, this method preferred over
|
||||||
|
// Set(string(data), path, value)
|
||||||
|
func SetBytes(json []byte, path string, value interface{}) ([]byte, error) {
|
||||||
|
return SetBytesOptions(json, path, value, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetBytesOptions sets a json value for the specified path with options.
|
||||||
|
// If working with bytes, this method preferred over
|
||||||
|
// SetOptions(string(data), path, value)
|
||||||
|
func SetBytesOptions(json []byte, path string, value interface{},
|
||||||
|
opts *Options) ([]byte, error) {
|
||||||
|
var optimistic, inplace bool
|
||||||
|
if opts != nil {
|
||||||
|
optimistic = opts.Optimistic
|
||||||
|
inplace = opts.ReplaceInPlace
|
||||||
|
}
|
||||||
|
jstr := *(*string)(unsafe.Pointer(&json))
|
||||||
|
var res []byte
|
||||||
|
var err error
|
||||||
|
switch v := value.(type) {
|
||||||
|
default:
|
||||||
|
b, err := jsongo.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
raw := *(*string)(unsafe.Pointer(&b))
|
||||||
|
res, err = set(jstr, path, raw, false, false, optimistic, inplace)
|
||||||
|
case dtype:
|
||||||
|
res, err = set(jstr, path, "", false, true, optimistic, inplace)
|
||||||
|
case string:
|
||||||
|
res, err = set(jstr, path, v, true, false, optimistic, inplace)
|
||||||
|
case []byte:
|
||||||
|
raw := *(*string)(unsafe.Pointer(&v))
|
||||||
|
res, err = set(jstr, path, raw, true, false, optimistic, inplace)
|
||||||
|
case bool:
|
||||||
|
if v {
|
||||||
|
res, err = set(jstr, path, "true", false, false, optimistic, inplace)
|
||||||
|
} else {
|
||||||
|
res, err = set(jstr, path, "false", false, false, optimistic, inplace)
|
||||||
|
}
|
||||||
|
case int8:
|
||||||
|
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case int16:
|
||||||
|
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case int32:
|
||||||
|
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case int64:
|
||||||
|
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case uint8:
|
||||||
|
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case uint16:
|
||||||
|
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case uint32:
|
||||||
|
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case uint64:
|
||||||
|
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case float32:
|
||||||
|
res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
case float64:
|
||||||
|
res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64),
|
||||||
|
false, false, optimistic, inplace)
|
||||||
|
}
|
||||||
|
if err == errNoChange {
|
||||||
|
return json, nil
|
||||||
|
}
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRaw sets a raw json value for the specified path.
|
||||||
|
// This function works the same as Set except that the value is set as a
|
||||||
|
// raw block of json. This allows for setting premarshalled json objects.
|
||||||
|
func SetRaw(json, path, value string) (string, error) {
|
||||||
|
return SetRawOptions(json, path, value, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRawOptions sets a raw json value for the specified path with options.
|
||||||
|
// This furnction works the same as SetOptions except that the value is set
|
||||||
|
// as a raw block of json. This allows for setting premarshalled json objects.
|
||||||
|
func SetRawOptions(json, path, value string, opts *Options) (string, error) {
|
||||||
|
var optimistic bool
|
||||||
|
if opts != nil {
|
||||||
|
optimistic = opts.Optimistic
|
||||||
|
}
|
||||||
|
res, err := set(json, path, value, false, false, optimistic, false)
|
||||||
|
if err == errNoChange {
|
||||||
|
return json, nil
|
||||||
|
}
|
||||||
|
return string(res), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRawBytes sets a raw json value for the specified path.
|
||||||
|
// If working with bytes, this method preferred over
|
||||||
|
// SetRaw(string(data), path, value)
|
||||||
|
func SetRawBytes(json []byte, path string, value []byte) ([]byte, error) {
|
||||||
|
return SetRawBytesOptions(json, path, value, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRawBytesOptions sets a raw json value for the specified path with options.
|
||||||
|
// If working with bytes, this method preferred over
|
||||||
|
// SetRawOptions(string(data), path, value, opts)
|
||||||
|
func SetRawBytesOptions(json []byte, path string, value []byte,
|
||||||
|
opts *Options) ([]byte, error) {
|
||||||
|
jstr := *(*string)(unsafe.Pointer(&json))
|
||||||
|
vstr := *(*string)(unsafe.Pointer(&value))
|
||||||
|
var optimistic, inplace bool
|
||||||
|
if opts != nil {
|
||||||
|
optimistic = opts.Optimistic
|
||||||
|
inplace = opts.ReplaceInPlace
|
||||||
|
}
|
||||||
|
res, err := set(jstr, path, vstr, false, false, optimistic, inplace)
|
||||||
|
if err == errNoChange {
|
||||||
|
return json, nil
|
||||||
|
}
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type dtype struct{}
|
||||||
|
|
||||||
|
// Delete deletes a value from json for the specified path.
|
||||||
|
func Delete(json, path string) (string, error) {
|
||||||
|
return Set(json, path, dtype{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteBytes deletes a value from json for the specified path.
|
||||||
|
func DeleteBytes(json []byte, path string) ([]byte, error) {
|
||||||
|
return SetBytes(json, path, dtype{})
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue