WIP: Sync

.aegir.js

@@ -0,0 +1,12 @@
+'use strict'
+
+module.exports = {
+  karma: {
+    files: [{
+      pattern: 'test/fixtures/**/*.js',
+      watched: false,
+      served: true,
+      included: false
+    }]
+  }
+}

.gitignore

@@ -1 +1,2 @@
 node_modules
+crash.log

.travis.yml

@@ -5,17 +5,22 @@ matrix:
   include:
   - node_js: 6
     env:
-      - SAUCE=true
+      - SAUCE=false
       - CXX=g++-4.8
 
 # Make sure we have new NPM.
-before_install:
-  - npm install -g npm
+# before_install:
+#   - npm install -g npm
+
+before_script:
+  - export DISPLAY=:99.0
+  - sh -e /etc/init.d/xvfb start
 
 script:
   - npm test
 
 addons:
+  firefox: 'latest'
   apt:
     sources:
       - ubuntu-toolchain-r-test

README.md

@@ -26,9 +26,50 @@ Arguments:
 
 * `options` (object, defaults to [this](src/default-options.js)): with the following keys:
   * `ipfsOptions` (object). [IPFS options object](https://github.com/ipfs/js-ipfs#advanced-options-when-creating-an-ipfs-node).
-  * `heads` (LevelDown-compatible database that stores the heads)
+  * `log` (LevelDown-compatible database that stores the log)
+  * `sync` (boolean, defaults to `false`): EXPERIMENTAL! syncs data between nodes
+  * `ipfs` (IPFS object): an IPFS object instance. If you already can provide an IPFS object, pass it in here.
 
 
+# Default arguments
+
+You can create a constructor that curries some default arguments by using `IPFSLevel.defaults(options)` like this:
+
+```js
+const ipfsLevel = IPFSLevel.defaults({
+  log: someLevelDownLogDatabase
+})
+```
+
+## Sync
+
+TODO: Explain sync
+
+
+## With Levelup
+
+
+This default options feature may be useful if you want to pass a constructor into which you'll have no saying about the options, like on the Levelup constructor:
+
+```js
+const LevelUp = require('levleup')
+const Memdown = require('memdown') // any leveldown db will do for caching log entries
+const const IPFSLevel = require('ipfs-level').defaults({
+  log: Memdown('some-partition-name') // log database should be scoped to partition
+})
+
+const db = LevelUp({ db: IPFSLevel })
+// now you have a levelup db you can use
+```
+
+# Internals
+
+The internals are documented [here](docs/INTERNALS.md).
+
+# Test and debug
+
+This package uses [debug](https://github.com/visionmedia/debug#readme), so you can activate debug messages by setting the environment variable `DEBUG` to `ipfs-level:*`
+
 # License
 
 MIT

docs/INTERNALS.md

@@ -0,0 +1,55 @@
+# ipfs-level Internals
+
+# Local put
+
+When a node does a `db.put(key, value, callback)`, this is what happens inside that node:
+
+* the key and value are encoded into a single document. We'll call this the 'kv-doc'
+* This kv-doc is written onto IPFS (using the IPFS DAG API)
+* IPFS gives you a Content ID (CID) in return, which uniquely identifies this document.
+* The node retrieves the latest log entry (the latest HEAD) from the local log
+* A new log entry is created. This entry contains:
+  * `parent`: the CID from the latest HEAD
+  * `key`: the kv-doc key
+  * `cid`: the CID the kv-doc
+  * `clock`: the updated vector clock, where the entry for the current node was incremented
+* This new log entry is written into the IPFS, using the DAG API
+* IPFS gives you a CID for that log entry
+* The node uses the CID as a key to save the log entry into the log
+* The node saves the log entry into the log using a key derived from the kv-doc key. This way it's easy to retrieve the latest log entry for a given value.
+* The node saves the log entry under the key `HEAD`, to be able to later retrieve it
+
+# Remote update
+
+When a node gets a remote log update, the message contains the latest log entry (the remote HEAD), which contains:
+  * `parent`: the remote parent log entre
+  * `key`: the update this log entry pertains to
+  * `cid`: the CID of the remote kv-doc that originated this entry
+  * `clock`: the remote vector clock that this log entry generated
+
+Upon receiving this message, the node goes thtough these procedures:
+
+* Full log retrieval:
+  * If the node already contains the parent log entry of the remote log entry, nothing is done
+  * Otherwise, it retrieves the parent log entry and stores it in the log
+  * Repeats these steps until all log entries are here
+* Log sorting:
+  * Collects all the new log entries
+  * Sorts them in temporal order
+* Log entry processing: starting from the earliest log entry, for each entry:
+  * retrieves the latest local log entry for that key
+  * compares the remote and local vector clock:
+  * if the remote vector clock precedes the local one: do nothing
+  * if the local vector clock precedes the remote one:
+    * point the latest entry for that key the new log entry
+  * if the local vector clock is concurrent with the remote one:
+    * here we have to deterministically pick one of the log entries as the winning one.
+    * pick the log entry with the highest CID
+    * if the remote log entry wins, point the latest entry for that key to this new entry.
+* HEAD update
+  * compare both the remote and the local head.
+  * if the local head precedes the remote one, point the new HEAD to the remote HEAD
+  * if the remote head precedes the local one, do nothing
+  * if both entries are concurrent, pick the one with the highest CID and set HEAD to that one.
+* HEAD broadcast
+  * Whenever there is a change in the HEAD pointer, use IPFS pubsub to broadcast that change to all interested nodes.

package.json

@@ -5,7 +5,10 @@
   "main": "src/index.js",
   "scripts": {
     "lint": "standard",
-    "test": "npm run lint && mocha --timeout=10000"
+    "test": "npm run compile:tests && npm run test:browser",
+    "test:browser": "aegir-test browser --dom",
+    "test:node": "mocha test/leveldown.js --timeout=20000 && mocha test/iterator.js --timeout=20000 && mocha test/sync.js --timeout=20000",
+    "compile:tests": "browserify test/abstract-leveldown.js -o test/abstract-leveldown-generated.spec.js --debug"
   },
   "repository": {
     "type": "git",
@@ -25,15 +28,28 @@
   },
   "homepage": "https://github.com/pgte/ipfs-level#readme",
   "devDependencies": {
+    "aegir": "^11.0.2",
+    "browserify": "^14.4.0",
     "chai": "^3.5.0",
     "dirty-chai": "^1.2.2",
+    "memdown": "^1.2.4",
     "mocha": "^3.4.1",
-    "standard": "^10.0.2"
+    "rimraf": "^2.6.1",
+    "standard": "^10.0.2",
+    "tape": "^4.6.3"
   },
   "dependencies": {
     "abstract-leveldown": "^2.6.1",
     "async": "^2.4.1",
+    "backoff": "^2.5.0",
+    "debug": "^2.6.8",
     "deep-assign": "^2.0.0",
-    "ipfs": "^0.23.1"
+    "hyperdiff": "^2.0.2",
+    "ipfs": "^0.24.0",
+    "lodash.clonedeep": "^4.5.0",
+    "vectorclock": "0.0.0"
+  },
+  "browser": {
+    "./test/utils/create-repo-node.js": "./test/utils/create-repo-browser.js"
   }
 }

src/decoding.js

@@ -0,0 +1,24 @@
+'use strict'
+
+function decode (str) {
+  if (Buffer.isBuffer(str)) {
+    str = str.toString()
+  }
+  if (typeof str === 'object') {
+    return str
+  }
+  return JSON.parse(str)
+}
+
+module.exports = function decoding (callback) {
+  if (typeof callback !== 'function') {
+    throw new Error('callback is not a function')
+  }
+  return (err, str) => {
+    if (err && err.message === 'NotFound') {
+      callback(null, undefined)
+    } else {
+      callback(err, str && decode(str))
+    }
+  }
+}

src/encode.js

@@ -6,10 +6,3 @@ exports.kv = (key, value, options) => {
     value: value
   }
 }
-
-exports.deleted = (key) => {
-  return {
-    key: key,
-    deleted: true
-  }
-}

src/index.js

@@ -1,5 +1,11 @@
 'use strict'
 
-const IPFSLeveldown = require('./ipfs-leveldown')
+const merge = require('deep-assign')
 
-module.exports = (partition, options) => new IPFSLeveldown(partition, options)
+const IPFSLevel = require('./ipfs-level')
+
+exports = module.exports = (partition, options) => new IPFSLevel(partition, options)
+exports.defaults = (defaultOptions) => (partition, options) => {
+  const opts = merge({}, defaultOptions, options || {})
+  return new IPFSLevel(partition, opts)
+}