Add startup testing for Wasm Pinecone build (#1910)
* Only include go-sqlite3 on the relevant binaries * The driver name is always sqlite3 now * Update to matrix-org/go-sqlite3-js@e537baa * Add initial Wasm test harness * Upgrade go-sqlite3-js This fixes an error about semicolons in single statements. * Add browser-like WebSocket API for testing * Upgrade go-sqlite3-js This upgrade includes printing panic messages next to stacks. * Run for all PRs targeting any branch * Use manual Node caching * Temporarily run for all pushes * Use npm ci instead of install * Use HTTPS auth for repo packages * Match path style from build.sh * update utp Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>main
parent
f63068df3b
commit
034f5ae3f1
|
@ -0,0 +1,49 @@
|
|||
name: WebAssembly
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16.5
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
|
||||
- name: Install Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 14
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-
|
||||
|
||||
- name: Reconfigure Git to use HTTPS auth for repo packages
|
||||
run: >
|
||||
git config --global url."https://github.com/".insteadOf
|
||||
ssh://git@github.com/
|
||||
|
||||
- name: Install test dependencies
|
||||
working-directory: ./test/wasm
|
||||
run: npm ci
|
||||
|
||||
- name: Test
|
||||
run: ./test-dendritejs.sh
|
|
@ -3,6 +3,9 @@
|
|||
# Hidden files
|
||||
.*
|
||||
|
||||
# Allow GitHub config
|
||||
!.github
|
||||
|
||||
# Downloads
|
||||
/.downloads
|
||||
|
||||
|
@ -36,6 +39,7 @@ _testmain.go
|
|||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
*.wasm
|
||||
|
||||
# Generated keys
|
||||
*.pem
|
||||
|
@ -53,3 +57,6 @@ dendrite.yaml
|
|||
|
||||
# Generated code
|
||||
cmd/dendrite-demo-yggdrasil/embed/fs*.go
|
||||
|
||||
# Test dependencies
|
||||
test/wasm/node_modules
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/sh -eu
|
||||
|
||||
export GIT_COMMIT=$(git rev-list -1 HEAD) && \
|
||||
GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs
|
||||
GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone
|
||||
|
|
2
build.sh
2
build.sh
|
@ -21,4 +21,4 @@ mkdir -p bin
|
|||
|
||||
CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
|
||||
|
||||
CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
|
||||
CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone
|
||||
|
|
|
@ -144,6 +144,13 @@ func generateKey() ed25519.PrivateKey {
|
|||
}
|
||||
|
||||
func main() {
|
||||
startup()
|
||||
|
||||
// We want to block forever to let the fetch and libp2p handler serve the APIs
|
||||
select {}
|
||||
}
|
||||
|
||||
func startup() {
|
||||
sk := generateKey()
|
||||
pk := sk.Public().(ed25519.PublicKey)
|
||||
|
||||
|
@ -251,7 +258,4 @@ func main() {
|
|||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// We want to block forever to let the fetch and libp2p handler serve the APIs
|
||||
select {}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,25 @@
|
|||
// Copyright 2021 The Matrix.org Foundation C.I.C.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build wasm
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStartup(t *testing.T) {
|
||||
startup()
|
||||
}
|
15
docs/p2p.md
15
docs/p2p.md
|
@ -2,14 +2,23 @@
|
|||
|
||||
These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
|
||||
|
||||
|
||||
### Dendrite
|
||||
|
||||
#### Build
|
||||
|
||||
- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
|
||||
- Build it and copy assets to riot-web.
|
||||
```
|
||||
$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
|
||||
$ cp main.wasm ../riot-web/src/vector/dendrite.wasm
|
||||
$ ./build-dendritejs.sh
|
||||
$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
|
||||
```
|
||||
|
||||
#### Test
|
||||
|
||||
To check that the Dendrite side is working well as Wasm, you can run the
|
||||
Wasm-specific tests:
|
||||
```
|
||||
$ ./test-dendritejs.sh
|
||||
```
|
||||
|
||||
### Rendezvous
|
||||
|
|
2
go.mod
2
go.mod
|
@ -37,7 +37,7 @@ require (
|
|||
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
|
||||
github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b
|
||||
github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
|
||||
github.com/opentracing/opentracing-go v1.2.0
|
||||
|
|
6
go.sum
6
go.sum
|
@ -118,9 +118,7 @@ github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgw
|
|||
github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo=
|
||||
github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw=
|
||||
github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y=
|
||||
github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw=
|
||||
github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
|
||||
github.com/anacrolix/sync v0.2.0 h1:oRe22/ZB+v7v/5Mbc4d2zE0AXEZy0trKyKLjqYOt6tY=
|
||||
github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g=
|
||||
github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
|
||||
github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk=
|
||||
|
@ -1186,8 +1184,8 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/
|
|||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b h1:XNm+Ks3bVziRJxcMaIbzumWEw7l52z9Rek6cMHgln1g=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b/go.mod h1:ylsx0342RjGHjOoVKhR/wz/7Lhiusonihfj4QLxEMcU=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2 h1:txJOiDxsypF8RbzbcyOD3ovip+uUWNZE/Zo7qLdARZQ=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh -eu
|
||||
|
||||
GOOS=js GOARCH=wasm go test -v -exec "$(pwd)/test/wasm/index.js" ./cmd/dendritejs-pinecone
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/*
|
||||
Copyright 2021 The Matrix.org Foundation C.I.C.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const childProcess = require('child_process');
|
||||
|
||||
(async function() {
|
||||
// sql.js
|
||||
const initSqlJs = require('sql.js');
|
||||
await initSqlJs().then(SQL => {
|
||||
global._go_sqlite = SQL;
|
||||
console.log("Loaded sqlite")
|
||||
});
|
||||
// dendritejs expects to write to `/idb` so we create that here
|
||||
// Since this is testing only, we use the default in-memory FS
|
||||
global._go_sqlite.FS.mkdir("/idb");
|
||||
|
||||
// WebSocket
|
||||
const WebSocket = require('isomorphic-ws');
|
||||
global.WebSocket = WebSocket;
|
||||
|
||||
// Load the generic Go Wasm exec helper inline to trigger built-in run call
|
||||
// This approach avoids copying `wasm_exec.js` into the repo, which is nice
|
||||
// to aim for since it can differ between Go versions.
|
||||
const goRoot = await new Promise((resolve, reject) => {
|
||||
childProcess.execFile('go', ['env', 'GOROOT'], (err, out) => {
|
||||
if (err) {
|
||||
reject("Can't find go");
|
||||
}
|
||||
resolve(out.trim());
|
||||
});
|
||||
});
|
||||
const execPath = path.join(goRoot, 'misc/wasm/wasm_exec.js');
|
||||
const execCode = fs.readFileSync(execPath, 'utf8');
|
||||
eval(execCode);
|
||||
})();
|
|
@ -0,0 +1,67 @@
|
|||
{
|
||||
"name": "wasm",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"isomorphic-ws": "^4.0.1",
|
||||
"sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
|
||||
"ws": "^7.5.2"
|
||||
}
|
||||
},
|
||||
"node_modules/isomorphic-ws": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
|
||||
"integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
|
||||
"peerDependencies": {
|
||||
"ws": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/sql.js": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
|
||||
"integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "7.5.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
|
||||
"integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
|
||||
"engines": {
|
||||
"node": ">=8.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": "^5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"isomorphic-ws": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
|
||||
"integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
|
||||
"requires": {}
|
||||
},
|
||||
"sql.js": {
|
||||
"version": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
|
||||
"integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
|
||||
"from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae"
|
||||
},
|
||||
"ws": {
|
||||
"version": "7.5.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
|
||||
"integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
|
||||
"requires": {}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"dependencies": {
|
||||
"isomorphic-ws": "^4.0.1",
|
||||
"sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
|
||||
"ws": "^7.5.2"
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue