1 files changed, 604 insertions, 174 deletions

diff --git a/data.c b/data.c
index 49f215b..51ee2c7 100644
--- a/data.c
+++ b/data.c
@@ -6,6 +6,7 @@
 #include "xs_json.h"
 #include "xs_openssl.h"
 #include "xs_glob.h"
+#include "xs_set.h"
 
 #include "snac.h"
 
@@ -14,12 +15,12 @@
 #include <sys/file.h>
 #include <fcntl.h>
 
-double db_layout = 2.1;
+double db_layout = 2.5;
 
 
 int db_upgrade(d_char **error);
 
-int srv_open(char *basedir)
+int srv_open(char *basedir, int auto_upgrade)
 /* opens a server */
 {
     int ret = 0;
@@ -69,7 +70,14 @@ int srv_open(char *basedir)
                     error = xs_fmt("DEBUG level set to %d from environment", dbglevel);
                 }
 
-                ret = db_upgrade(&error);
+                if (auto_upgrade)
+                    ret = db_upgrade(&error);
+                else {
+                    if (xs_number_get(xs_dict_get(srv_config, "layout")) < db_layout)
+                        error = xs_fmt("ERROR: disk layout changed - execute 'snac upgrade' first");
+                    else
+                        ret = 1;
+                }
             }
 
         }
@@ -178,24 +186,215 @@ d_char *user_list(void)
 }
 
 
-double mtime(char *fn)
-/* returns the mtime of a file or directory, or 0.0 */
+double mtime_nl(const char *fn, int *n_link)
+/* returns the mtime and number of links of a file or directory, or 0.0 */
 {
     struct stat st;
     double r = 0.0;
+    int n = 0;
 
-    if (fn && stat(fn, &st) != -1)
-        r = (double)st.st_mtim.tv_sec;
+    if (fn && stat(fn, &st) != -1) {
+        r = (double) st.st_mtim.tv_sec;
+        n = st.st_nlink;
+    }
+
+    if (n_link)
+        *n_link = n;
 
     return r;
 }
 
 
-/** object database 2.1+ **/
+/** database 2.1+ **/
+
+/** indexes **/
+
+int index_add_md5(const char *fn, const char *md5)
+/* adds an md5 to an index */
+{
+    int status = 201; /* Created */
+    FILE *f;
+
+    if ((f = fopen(fn, "a")) != NULL) {
+        flock(fileno(f), LOCK_EX);
+
+        /* ensure the position is at the end after getting the lock */
+        fseek(f, 0, SEEK_END);
+
+        fprintf(f, "%s\n", md5);
+        fclose(f);
+    }
+    else
+        status = 500;
+
+    return status;
+}
+
+
+int index_add(const char *fn, const char *id)
+/* adds an id to an index */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return index_add_md5(fn, md5);
+}
+
+
+int index_del(const char *fn, const char *md5)
+/* deletes an md5 from an index */
+{
+    int status = 404;
+    FILE *i, *o;
+
+    if ((i = fopen(fn, "r")) != NULL) {
+        flock(fileno(i), LOCK_EX);
+
+        xs *nfn = xs_fmt("%s.new", fn);
+        char line[256];
+
+        if ((o = fopen(nfn, "w")) != NULL) {
+            while (fgets(line, sizeof(line), i) != NULL) {
+                line[32] = '\0';
+                if (memcmp(line, md5, 32) != 0)
+                    fprintf(o, "%s\n", line);
+            }
+
+            fclose(o);
+
+            xs *ofn = xs_fmt("%s.bak", fn);
+
+            link(fn, ofn);
+            rename(nfn, fn);
+        }
+        else
+            status = 500;
+
+        fclose(i);
+    }
+    else
+        status = 500;
+
+    return status;
+}
+
+
+int index_in_md5(const char *fn, const char *md5)
+/* checks if the md5 is already in the index */
+{
+    FILE *f;
+    int ret = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+
+        while (!ret && fgets(line, sizeof(line), f) != NULL) {
+            line[32] = '\0';
+
+            if (strcmp(line, md5) == 0)
+                ret = 1;
+        }
+
+        fclose(f);
+    }
+
+    return ret;
+}
+
+
+int index_in(const char *fn, const char *id)
+/* checks if the object id is already in the index */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return index_in_md5(fn, md5);
+}
+
+
+int index_first(const char *fn, char *line, int size)
+/* reads the first entry of an index */
+{
+    FILE *f;
+    int ret = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        if (fgets(line, size, f) != NULL) {
+            line[32] = '\0';
+            ret = 1;
+        }
+
+        fclose(f);
+    }
+
+    return ret;
+}
+
+
+d_char *index_list(const char *fn, int max)
+/* returns an index as a list */
+{
+    d_char *list = NULL;
+    FILE *f;
+    int n = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+        list = xs_list_new();
+
+        while (n < max && fgets(line, sizeof(line), f) != NULL) {
+            line[32] = '\0';
+            list = xs_list_append(list, line);
+            n++;
+        }
+
+        fclose(f);
+    }
+
+    return list;
+}
+
+
+d_char *index_list_desc(const char *fn, int max)
+/* returns an index as a list, in reverse order */
+{
+    d_char *list = NULL;
+    FILE *f;
+    int n = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+        list = xs_list_new();
+
+        /* move to the end minus one entry */
+        if (!fseek(f, 0, SEEK_END) && !fseek(f, -33, SEEK_CUR)) {
+            while (n < max && fgets(line, sizeof(line), f) != NULL) {
+                line[32] = '\0';
+                list = xs_list_append(list, line);
+                n++;
+
+                /* move backwards 2 entries */
+                if (fseek(f, -66, SEEK_CUR) == -1)
+                    break;
+            }
+        }
+
+        fclose(f);
+    }
+
+    return list;
+}
+
+
+/** objects **/
 
 d_char *_object_fn_by_md5(const char *md5)
 {
-    xs *bfn = xs_fmt("%s/object/%c%c/", srv_basedir, md5[0], md5[1]);
+    xs *bfn = xs_fmt("%s/object/%c%c", srv_basedir, md5[0], md5[1]);
 
     mkdir(bfn, 0755);
 
@@ -203,19 +402,18 @@ d_char *_object_fn_by_md5(const char *md5)
 }
 
 
-d_char *_object_fn_by_id(const char *id)
+d_char *_object_fn(const char *id)
 {
     xs *md5 = xs_md5_hex(id, strlen(id));
-
     return _object_fn_by_md5(md5);
 }
 
 
-int object_get(const char *id, d_char **obj, const char *type)
-/* returns a loaded object, optionally of the requested type */
+int object_get_by_md5(const char *md5, d_char **obj, const char *type)
+/* returns a stored object, optionally of the requested type */
 {
     int status = 404;
-    xs *fn     = _object_fn_by_id(id);
+    xs *fn     = _object_fn_by_md5(md5);
     FILE *f;
 
     if ((f = fopen(fn, "r")) != NULL) {
@@ -247,13 +445,27 @@ int object_get(const char *id, d_char **obj, const char *type)
 }
 
 
-int object_add(const char *id, d_char *obj)
+int object_get(const char *id, d_char **obj, const char *type)
+/* returns a stored object, optionally of the requested type */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return object_get_by_md5(md5, obj, type);
+}
+
+
+int _object_add(const char *id, d_char *obj, int ow)
 /* stores an object */
 {
     int status = 201; /* Created */
-    xs *fn     = _object_fn_by_id(id);
+    xs *fn     = _object_fn(id);
     FILE *f;
 
+    if (!ow && mtime(fn) > 0.0) {
+        /* object already here */
+        srv_debug(0, xs_fmt("object_add object already here %s", id));
+        return 204; /* No content */
+    }
+
     if ((f = fopen(fn, "w")) != NULL) {
         flock(fileno(f), LOCK_EX);
 
@@ -261,100 +473,241 @@ int object_add(const char *id, d_char *obj)
 
         fwrite(j, strlen(j), 1, f);
         fclose(f);
+
+        /* does this object has a parent? */
+        char *in_reply_to = xs_dict_get(obj, "inReplyTo");
+
+        if (!xs_is_null(in_reply_to) && *in_reply_to) {
+            /* update the children index of the parent */
+            xs *c_idx = _object_fn(in_reply_to);
+
+            c_idx = xs_replace_i(c_idx, ".json", "_c.idx");
+            index_add(c_idx, id);
+
+            srv_debug(0, xs_fmt("object_add added child %s to %s", id, c_idx));
+
+            /* create a one-element index with the parent */
+            xs *p_idx = xs_replace(fn, ".json", "_p.idx");
+            index_add(p_idx, in_reply_to);
+
+            srv_debug(0, xs_fmt("object_add added parent %s to %s", in_reply_to, p_idx));
+        }
     }
     else
         status = 500;
 
+    srv_debug(0, xs_fmt("object_add %s %s %d", id, fn, status));
+
     return status;
 }
 
 
-d_char *_follower_fn(snac *snac, char *actor)
+int object_add(const char *id, d_char *obj)
+/* stores an object */
 {
-    xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/followers/%s.json", snac->basedir, md5);
+    return _object_add(id, obj, 0);
 }
 
 
-int follower_add(snac *snac, char *actor, char *msg)
-/* adds a follower */
+int object_add_ow(const char *id, d_char *obj)
+/* stores an object (overwriting allowed) */
 {
-    int ret = 201; /* created */
-    xs *fn = _follower_fn(snac, actor);
-    FILE *f;
+    return _object_add(id, obj, 1);
+}
 
-    if ((f = fopen(fn, "w")) != NULL) {
-        xs *j = xs_json_dumps_pp(msg, 4);
 
-        fwrite(j, 1, strlen(j), f);
-        fclose(f);
+int object_del_by_md5(const char *md5)
+/* deletes an object by its md5 */
+{
+    int status = 404;
+    xs *fn     = _object_fn_by_md5(md5);
+
+    if (fn != NULL && unlink(fn) != -1) {
+        status = 200;
+
+        /* also delete associated indexes */
+        xs *spec  = xs_dup(fn);
+        spec      = xs_replace_i(spec, ".json", "*.idx");
+        xs *files = xs_glob(spec, 0, 0);
+        char *p, *v;
+
+        p = files;
+        while (xs_list_iter(&p, &v)) {
+            srv_debug(0, xs_fmt("object_del index %s", v));
+            unlink(v);
+        }
     }
-    else
-        ret = 500;
 
-    snac_debug(snac, 2, xs_fmt("follower_add %s %s", actor, fn));
+    srv_debug(0, xs_fmt("object_del %s %d", fn, status));
+
+    return status;
+}
+
+
+int object_del(const char *id)
+/* deletes an object */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return object_del_by_md5(md5);
+}
+
+
+int object_del_if_unref(const char *id)
+/* deletes an object if its n_links < 2 */
+{
+    xs *fn = _object_fn(id);
+    int n_links;
+    int ret = 0;
+
+    if (mtime_nl(fn, &n_links) > 0.0 && n_links < 2)
+        ret = object_del(id);
 
     return ret;
 }
 
 
-int follower_del(snac *snac, char *actor)
-/* deletes a follower */
+d_char *object_children(const char *id)
+/* returns the list of an object's children */
+{
+    xs *fn = _object_fn(id);
+
+    fn = xs_replace_i(fn, ".json", "_c.idx");
+
+    return index_list(fn, XS_ALL);
+}
+
+
+int object_admire(const char *id, const char *actor, int like)
+/* actor likes or announces this object */
 {
     int status = 200;
-    xs *fn = _follower_fn(snac, actor);
+    xs *fn     = _object_fn(id);
 
-    if (fn != NULL)
-        unlink(fn);
-    else
-        status = 404;
+    fn = xs_replace_i(fn, ".json", like ? "_l.idx" : "_a.idx");
 
-    snac_debug(snac, 2, xs_fmt("follower_del %s %s", actor, fn));
+    if (!index_in(fn, actor)) {
+        status = index_add(fn, actor);
+
+        srv_debug(0, xs_fmt("object_admire (%s) %s %s", like ? "Like" : "Announce", actor, fn));
+    }
 
     return status;
 }
 
 
-int follower_check(snac *snac, char *actor)
-/* checks if someone is a follower */
+int _object_user_cache(snac *snac, const char *id, const char *cachedir, int del)
+/* adds or deletes from a user cache */
 {
-    xs *fn = _follower_fn(snac, actor);
+    xs *ofn = _object_fn(id);
+    xs *l   = xs_split(ofn, "/");
+    xs *cfn = xs_fmt("%s/%s/%s", snac->basedir, cachedir, xs_list_get(l, -1));
+    xs *idx = xs_fmt("%s/%s.idx", snac->basedir, cachedir);
+    int ret;
+
+    if (del) {
+        if ((ret = unlink(cfn)) != -1)
+            index_del(idx, id);
+    }
+    else {
+        index_add(idx, id);
+        ret = link(ofn, cfn);
+    }
 
-    return !!(mtime(fn) != 0.0);
+    return ret;
+}
+
+
+int object_user_cache_add(snac *snac, const char *id, const char *cachedir)
+/* caches an object into a user cache */
+{
+    return _object_user_cache(snac, id, cachedir, 0);
+}
+
+
+int object_user_cache_del(snac *snac, const char *id, const char *cachedir)
+/* deletes an object from a user cache */
+{
+    return _object_user_cache(snac, id, cachedir, 1);
+}
+
+
+int object_user_cache_in(snac *snac, const char *id, const char *cachedir)
+/* checks if an object is stored in a cache */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    xs *cfn = xs_fmt("%s/%s/%s.json", snac->basedir, cachedir, md5);
+
+    return !!(mtime(cfn) != 0.0);
+}
+
+
+d_char *object_user_cache_list(snac *snac, const char *cachedir, int max)
+/* returns the objects in a cache as a list */
+{
+    xs *idx = xs_fmt("%s/%s.idx", snac->basedir, cachedir);
+    return index_list(idx, max);
+}
+
+
+/** specialized functions **/
+
+/** followers **/
+
+int follower_add(snac *snac, const char *actor)
+/* adds a follower */
+{
+    int ret = object_user_cache_add(snac, actor, "followers");
+
+    snac_debug(snac, 2, xs_fmt("follower_add %s %s", actor));
+
+    return ret == -1 ? 500 : 200;
+}
+
+
+int follower_del(snac *snac, const char *actor)
+/* deletes a follower */
+{
+    int ret = object_user_cache_del(snac, actor, "followers");
+
+    snac_debug(snac, 2, xs_fmt("follower_del %s %s", actor));
+
+    return ret == -1 ? 404 : 200;
+}
+
+
+int follower_check(snac *snac, const char *actor)
+/* checks if someone is a follower */
+{
+    return object_user_cache_in(snac, actor, "followers");
 }
 
 
 d_char *follower_list(snac *snac)
 /* returns the list of followers */
 {
-    xs *spec = xs_fmt("%s/followers/" "*.json", snac->basedir);
-    xs *glist = xs_glob(spec, 0, 0);
+    xs *list      = object_user_cache_list(snac, "followers", XS_ALL);
+    d_char *fwers = xs_list_new();
     char *p, *v;
-    d_char *list = xs_list_new();
 
-    /* iterate the list of files */
-    p = glist;
+    /* resolve the list of md5 to be a list of actors */
+    p = list;
     while (xs_list_iter(&p, &v)) {
-        FILE *f;
-
-        /* load the follower data */
-        if ((f = fopen(v, "r")) != NULL) {
-            xs *j = xs_readall(f);
-            fclose(f);
+        xs *a_obj = NULL;
 
-            if (j != NULL) {
-                xs *o = xs_json_loads(j);
+        if (valid_status(object_get_by_md5(v, &a_obj, NULL))) {
+            char *actor = xs_dict_get(a_obj, "id");
 
-                if (o != NULL)
-                    list = xs_list_append(list, o);
-            }
+            if (!xs_is_null(actor))
+                fwers = xs_list_append(fwers, actor);
         }
     }
 
-    return list;
+    return fwers;
 }
 
 
+/** timeline **/
+
 double timeline_mtime(snac *snac)
 {
     xs *fn = xs_fmt("%s/timeline", snac->basedir);
@@ -437,6 +790,13 @@ int timeline_del(snac *snac, char *id)
         ret = 200;
     }
 
+    /* delete from the user's caches */
+    object_user_cache_del(snac, id, "public");
+    object_user_cache_del(snac, id, "private");
+
+    /* try to delete the object if it's not used elsewhere */
+    object_del_if_unref(id);
+
     return ret;
 }
 
@@ -511,22 +871,11 @@ int _timeline_write(snac *snac, char *id, char *msg, char *parent, char *referre
 
         if (pfn != NULL && (f = fopen(pfn, "r")) != NULL) {
             xs *j;
-            char *v;
 
             j = xs_readall(f);
             fclose(f);
 
             p_msg = xs_json_loads(j);
-
-            if ((v = xs_dict_get(p_msg, "_snac")) != NULL) {
-                /* is parent hidden? */
-                if ((v = xs_dict_get(v, "hidden")) && xs_type(v) == XSTYPE_TRUE) {
-                    snac_debug(snac, 1,
-                        xs_fmt("_timeline_write dropping due to hidden parent %s (%s)", id, parent));
-
-                    return 0;
-                }
-            }
         }
     }
 
@@ -648,6 +997,16 @@ int _timeline_write(snac *snac, char *id, char *msg, char *parent, char *referre
 }
 
 
+void timeline_update_indexes(snac *snac, const char *id)
+/* updates the indexes */
+{
+    object_user_cache_add(snac, id, "private");
+
+    if (xs_startswith(id, snac->actor))
+        object_user_cache_add(snac, id, "public");
+}
+
+
 int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer)
 /* adds a message to the timeline */
 {
@@ -680,13 +1039,61 @@ int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer
 
     msg = xs_dict_set(msg, "_snac", md);
 
-    if ((ret = _timeline_write(snac, id, msg, parent, referrer)))
+    if ((ret = _timeline_write(snac, id, msg, parent, referrer))) {
         snac_debug(snac, 1, xs_fmt("timeline_add %s", id));
 
+        object_add(id, o_msg);
+        timeline_update_indexes(snac, id);
+    }
+
     return ret;
 }
 
 
+d_char *timeline_top_level(snac *snac, d_char *list)
+/* returns the top level md5 entries from this index */
+{
+    d_char *tl = xs_list_new();
+    xs_set seen;
+    char *p, *v;
+
+    xs_set_init(&seen);
+
+    p = list;
+    while (xs_list_iter(&p, &v)) {
+        char line[256] = "";
+
+        strcpy(line, v);
+
+        for (;;) {
+            char line2[256];
+            xs *fn = _object_fn_by_md5(line);
+            fn     = xs_replace_i(fn, ".json", "_p.idx");
+
+            /* if it doesn't have a parent, use this */
+            if (index_first(fn, line2, sizeof(line2)) == 0)
+                break;
+
+            xs *pfn = _object_fn_by_md5(line2);
+
+            /* well, there is a parent... but if it's not there, use this */
+            if (mtime(pfn) == 0.0)
+                break;
+
+            /* it's here! try again with its own parent */
+            strcpy(line, line2);
+        }
+
+        if (xs_set_add(&seen, line) == 1)
+            tl = xs_list_append(tl, line);
+    }
+
+    xs_set_free(&seen);
+
+    return tl;
+}
+
+
 void timeline_admire(snac *snac, char *id, char *admirer, int like)
 /* updates a timeline entry with a new admiration */
 {
@@ -734,52 +1141,16 @@ void timeline_admire(snac *snac, char *id, char *admirer, int like)
     }
     else
         snac_log(snac, xs_fmt("timeline_admire ignored for unknown object %s", id));
-}
-
-
-int timeline_hide(snac *snac, char *id, int hide)
-/* hides/unhides a timeline entry */
-{
-    int ret = 0;
-    xs *fn = _timeline_find_fn(snac, id);
-    FILE *f;
-
-    if (fn != NULL && (f = fopen(fn, "r")) != NULL) {
-        xs *s1   = xs_readall(f);
-        xs *msg  = xs_json_loads(s1);
-        xs *meta = xs_dup(xs_dict_get(msg, "_snac"));
-        xs *hdn  = xs_val_new(hide ? XSTYPE_TRUE : XSTYPE_FALSE);
-        char *p, *v;
-
-        fclose(f);
-
-        /* if it's already in this hidden state, we're done */
-        if ((v = xs_dict_get(meta, "hidden")) && xs_type(v) == xs_type(hdn))
-            return ret;
-
-        meta = xs_dict_set(meta, "hidden", hdn);
-        msg  = xs_dict_set(msg,  "_snac",  meta);
-
-        if ((f = fopen(fn, "w")) != NULL) {
-            xs *j1 = xs_json_dumps_pp(msg, 4);
-
-            fwrite(j1, strlen(j1), 1, f);
-            fclose(f);
 
-            snac_debug(snac, 1, xs_fmt("timeline_hide %d %s", hide, id));
+    object_admire(id, admirer, like);
+}
 
-            /* now hide the children */
-            p = xs_dict_get(meta, "children");
-            while (xs_list_iter(&p, &v))
-                timeline_hide(snac, v, hide);
 
-            ret = 1;
-        }
-    }
-
-    return ret;
-}
+/** following **/
 
+/* this needs special treatment and cannot use the object db as is,
+   with a link to a cached author, because we need the Follow object
+   in case we need to unfollow (Undo + original Follow) */
 
 d_char *_following_fn(snac *snac, char *actor)
 {
@@ -811,7 +1182,7 @@ int following_add(snac *snac, char *actor, char *msg)
 
 
 int following_del(snac *snac, char *actor)
-/* someone is no longer following us */
+/* we're not following this actor any longer */
 {
     xs *fn = _following_fn(snac, actor);
 
@@ -824,7 +1195,7 @@ int following_del(snac *snac, char *actor)
 
 
 int following_check(snac *snac, char *actor)
-/* checks if someone is following us */
+/* checks if we are following this actor */
 {
     xs *fn = _following_fn(snac, actor);
 
@@ -877,8 +1248,12 @@ d_char *following_list(snac *snac)
                 if (o != NULL) {
                     char *type = xs_dict_get(o, "type");
 
-                    if (!xs_is_null(type) && strcmp(type, "Accept") == 0)
-                        list = xs_list_append(list, o);
+                    if (!xs_is_null(type) && strcmp(type, "Accept") == 0) {
+                        char *actor = xs_dict_get(o, "actor");
+
+                        if (!xs_is_null(actor))
+                            list = xs_list_append(list, actor);
+                    }
                 }
             }
         }
@@ -891,7 +1266,7 @@ d_char *following_list(snac *snac)
 d_char *_muted_fn(snac *snac, char *actor)
 {
     xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/muted/%s.json", snac->basedir, md5);
+    return xs_fmt("%s/muted/%s", snac->basedir, md5);
 }
 
 
@@ -930,65 +1305,89 @@ int is_muted(snac *snac, char *actor)
 }
 
 
-d_char *_actor_fn(snac *snac, char *actor)
-/* returns the file name for an actor */
+d_char *_hidden_fn(snac *snac, const char *id)
 {
-    xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/actors/%s.json", snac->basedir, md5);
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return xs_fmt("%s/hidden/%s", snac->basedir, md5);
 }
 
 
-int actor_add(snac *snac, char *actor, char *msg)
-/* adds an actor */
+void hide(snac *snac, const char *id)
+/* hides a message tree */
 {
-    int ret = 201; /* created */
-    xs *fn = _actor_fn(snac, actor);
+    xs *fn = _hidden_fn(snac, id);
     FILE *f;
 
     if ((f = fopen(fn, "w")) != NULL) {
-        xs *j = xs_json_dumps_pp(msg, 4);
-
-        fwrite(j, 1, strlen(j), f);
+        fprintf(f, "%s\n", id);
         fclose(f);
+
+        snac_debug(snac, 2, xs_fmt("hidden %s %s", id, fn));
+
+        /* hide all the children */
+        xs *chld = object_children(id);
+        char *p, *v;
+
+        p = chld;
+        while (xs_list_iter(&p, &v)) {
+            xs *co = NULL;
+
+            /* resolve to get the id */
+            if (valid_status(object_get_by_md5(v, &co, NULL))) {
+                if ((v = xs_dict_get(co, "id")) != NULL)
+                    hide(snac, v);
+            }
+        }
     }
-    else
-        ret = 500;
+}
 
-    snac_debug(snac, 2, xs_fmt("actor_add %s %s", actor, fn));
 
-//    object_add(actor, msg);
+int is_hidden(snac *snac, const char *id)
+/* check is id is hidden */
+{
+    xs *fn = _hidden_fn(snac, id);
 
-    return ret;
+    return !!(mtime(fn) != 0.0);
+}
+
+
+int actor_add(snac *snac, const char *actor, d_char *msg)
+/* adds an actor */
+{
+    return object_add_ow(actor, msg);
 }
 
 
-int actor_get(snac *snac, char *actor, d_char **data)
+int actor_get(snac *snac, const char *actor, d_char **data)
 /* returns an already downloaded actor */
 {
-    xs *fn = _actor_fn(snac, actor);
-    double t;
-    double max_time;
-    int status;
-    FILE *f;
+    int status = 200;
+    char *d;
 
     if (strcmp(actor, snac->actor) == 0) {
+        /* this actor */
         if (data)
             *data = msg_actor(snac);
 
-        return 200;
+        return status;
     }
 
-    t = mtime(fn);
+    /* read the object */
+    if (!valid_status(status = object_get(actor, &d, NULL)))
+        return status;
 
-    /* no mtime? there is nothing here */
-    if (t == 0.0)
-        return 404;
+    if (data)
+        *data = d;
+
+    xs *fn = _object_fn(actor);
+    double max_time;
 
     /* maximum time for the actor data to be considered stale */
     max_time = 3600.0 * 36.0;
 
-    if (t + max_time < (double) time(NULL)) {
+    if (mtime(fn) + max_time < (double) time(NULL)) {
         /* actor data exists but also stinks */
+        FILE *f;
 
         if ((f = fopen(fn, "a")) != NULL) {
             /* write a blank at the end to 'touch' the file */
@@ -998,22 +1397,6 @@ int actor_get(snac *snac, char *actor, d_char **data)
 
         status = 205; /* "205: Reset Content" "110: Response Is Stale" */
     }
-    else {
-        /* it's still valid */
-        status = 200;
-    }
-
-    if (data) {
-        if ((f = fopen(fn, "r")) != NULL) {
-            xs *j = xs_readall(f);
-
-            fclose(f);
-
-            *data = xs_json_loads(j);
-        }
-        else
-            status = 500;
-    }
 
     return status;
 }
@@ -1033,7 +1416,7 @@ int static_get(snac *snac, const char *id, d_char **data, int *size)
     FILE *f;
     int status = 404;
 
-    *size = 0xfffffff;
+    *size = XS_ALL;
 
     if ((f = fopen(fn, "rb")) != NULL) {
         *data = xs_read(f, size);
@@ -1121,6 +1504,8 @@ d_char *history_list(snac *snac)
 }
 
 
+/** the queue **/
+
 static int _enqueue_put(char *fn, char *msg)
 /* writes safely to the queue */
 {
@@ -1267,38 +1652,81 @@ d_char *dequeue(snac *snac, char *fn)
 }
 
 
+/** the purge **/
+
+static void _purge_file(const char *fn, time_t mt)
+/* purge fn if it's older than days */
+{
+    if (mtime(fn) < mt) {
+        /* older than the minimum time: delete it */
+        unlink(fn);
+        srv_debug(1, xs_fmt("purged %s", fn));
+    }
+}
+
+
 static void _purge_subdir(snac *snac, const char *subdir, int days)
 /* purges all files in subdir older than days */
 {
     if (days) {
         time_t mt = time(NULL) - days * 24 * 3600;
-        xs *spec  = xs_fmt("%s/%s/" "*.json", snac->basedir, subdir);
+        xs *spec  = xs_fmt("%s/%s/" "*", snac->basedir, subdir);
         xs *list  = xs_glob(spec, 0, 0);
         char *p, *v;
 
         p = list;
-        while (xs_list_iter(&p, &v)) {
-            if (mtime(v) < mt) {
-                /* older than the minimum time: delete it */
-                unlink(v);
-                snac_debug(snac, 1, xs_fmt("purged %s", v));
+        while (xs_list_iter(&p, &v))
+            _purge_file(v, mt);
+    }
+}
+
+
+void purge_server(void)
+/* purge global server data */
+{
+    int tpd = xs_number_get(xs_dict_get(srv_config, "timeline_purge_days"));
+    xs *spec = xs_fmt("%s/object/??", srv_basedir);
+    xs *dirs = xs_glob(spec, 0, 0);
+    char *p, *v;
+
+    time_t mt = time(NULL) - tpd * 24 * 3600;
+
+    p = dirs;
+    while (xs_list_iter(&p, &v)) {
+        xs *spec2 = xs_fmt("%s/" "*.json", v);
+        xs *files = xs_glob(spec2, 0, 0);
+        char *p2, *v2;
+
+        p2 = files;
+        while (xs_list_iter(&p2, &v2)) {
+            int n_link;
+
+            /* old and with no hard links? */
+            if (mtime_nl(v2, &n_link) < mt && n_link < 2) {
+                xs *s1    = xs_replace(v2, ".json", "");
+                xs *l     = xs_split(s1, "/");
+                char *md5 = xs_list_get(l, -1);
+
+                object_del_by_md5(md5);
             }
         }
     }
 }
 
 
-void purge(snac *snac)
-/* do the purge */
+void purge_user(snac *snac)
+/* do the purge for this user */
 {
     int days;
 
     days = xs_number_get(xs_dict_get(srv_config, "timeline_purge_days"));
     _purge_subdir(snac, "timeline", days);
-    _purge_subdir(snac, "actors", days);
+    _purge_subdir(snac, "hidden", days);
+    _purge_subdir(snac, "private", days);
 
     days = xs_number_get(xs_dict_get(srv_config, "local_purge_days"));
     _purge_subdir(snac, "local", days);
+    _purge_subdir(snac, "public", days);
 }
 
 
@@ -1312,8 +1740,10 @@ void purge_all(void)
     p = list;
     while (xs_list_iter(&p, &uid)) {
         if (user_open(&snac, uid)) {
-            purge(&snac);
+            purge_user(&snac);
             user_free(&snac);
         }
     }
+
+    purge_server();
 }