19 files changed, 888 insertions, 227 deletions

diff --git a/README.md b/README.md
index 6f5d5cd..1b7dc50 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@ A simple, minimalistic ActivityPub instance
 - Lightweight, minimal dependencies
 - Extensive support of ActivityPub operations, e.g. write public notes, follow users, be followed, reply to the notes of others, admire wonderful content (like or boost), write private messages...
 - Simple but effective web interface
+- Multiuser
 - Easily-accessed MUTE button to silence morons
 - Tested interoperability with related software
 - No database needed
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index a0706c8..1d8a3ea 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,5 +1,13 @@
 # Release Notes
 
+## 2.13
+
+A big disk layout rework, to make it more efficient when timelines get very big. Please take note that you must run `snac upgrade` when you install this version over an already existing one.
+
+Fixed HTML loose close tag.
+
+Fixed bug when closing sendmail pipe.
+
 ## 2.12
 
 Fixed some bugs triggered when a GET query does not have an `Accept:` header.
diff --git a/TODO.md b/TODO.md
index c80387d..1145f88 100644
--- a/TODO.md
+++ b/TODO.md
@@ -2,7 +2,11 @@
 
 ## Open
 
-Dropping on input those messages that have their parent hidden is not a good idea, as children of *these* dropped messages will pass unharmed.
+Add an ?skip=NNN parameter to the admin page, to see older timeline.
+
+Add support for Edit + Note.
+
+Add domain/subdomain flexibility according to https://codeberg.org/grunfink/snac2/issues/3
 
 Add support for uploading the avatar, instead of needing an URL to an image. As a kludgy workaround, you can post something with an attached image, copy the auto-generated URL and use it. You can even delete the post, as attached images are never deleted (I said it was kludgy).
 
@@ -177,3 +181,5 @@ Add a purge timeout also for the local timeline (2022-11-12T08:32:56+0100).
 Add a switch for sensitive posts (2022-11-16T12:17:50+0100).
 
 Add an RSS to the local timeline (2022-11-18T11:43:54+0100).
+
+Dropping on input those messages that have their parent hidden is not a good idea, as children of *these* dropped messages will pass unharmed (2022-11-28T11:34:56+0100).
diff --git a/activitypub.c b/activitypub.c
index 5f26f73..7f63310 100644
--- a/activitypub.c
+++ b/activitypub.c
@@ -9,6 +9,7 @@
 #include "xs_openssl.h"
 #include "xs_regex.h"
 #include "xs_time.h"
+#include "xs_set.h"
 
 #include "snac.h"
 
@@ -169,11 +170,13 @@ int send_to_actor(snac *snac, char *actor, char *msg, d_char **payload, int *p_s
 d_char *recipient_list(snac *snac, char *msg, int expand_public)
 /* returns the list of recipients for a message */
 {
-    d_char *list = xs_list_new();
     char *to = xs_dict_get(msg, "to");
     char *cc = xs_dict_get(msg, "cc");
+    xs_set rcpts;
     int n;
 
+    xs_set_init(&rcpts);
+
     char *lists[] = { to, cc, NULL };
     for (n = 0; lists[n]; n++) {
         char *l = lists[n];
@@ -192,45 +195,41 @@ d_char *recipient_list(snac *snac, char *msg, int expand_public)
             if (expand_public && strcmp(v, public_address) == 0) {
                 /* iterate the followers and add them */
                 xs *fwers = follower_list(snac);
-                char *fw;
+                char *actor;
 
                 char *p = fwers;
-                while (xs_list_iter(&p, &fw)) {
-                    char *actor = xs_dict_get(fw, "actor");
-
-                    if (xs_list_in(list, actor) == -1)
-                        list = xs_list_append(list, actor);
-                }
+                while (xs_list_iter(&p, &actor))
+                    xs_set_add(&rcpts, actor);
             }
             else
-            if (xs_list_in(list, v) == -1)
-                list = xs_list_append(list, v);
+                xs_set_add(&rcpts, v);
         }
     }
 
-    return list;
+    return xs_set_result(&rcpts);
 }
 
 
 d_char *inbox_list(snac *snac, char *msg)
 /* returns the list of inboxes that are recipients of this message */
 {
-    d_char *list = xs_list_new();
-    xs *rcpts    = recipient_list(snac, msg, 1);
+    xs *rcpts = recipient_list(snac, msg, 1);
+    xs_set inboxes;
     char *p, *v;
 
+    xs_set_init(&inboxes);
+
     p = rcpts;
     while (xs_list_iter(&p, &v)) {
         xs *inbox;
 
         if ((inbox = get_actor_inbox(snac, v)) != NULL) {
             /* add the inbox if it's not already there */
-            if (xs_list_in(list, inbox) == -1)
-                list = xs_list_append(list, inbox);
+            xs_set_add(&inboxes, inbox);
         }
     }
 
-    return list;
+    return xs_set_result(&inboxes);
 }
 
 
@@ -824,7 +823,7 @@ int process_message(snac *snac, char *msg, char *req)
 
         timeline_add(snac, xs_dict_get(f_msg, "id"), f_msg, NULL, NULL);
 
-        follower_add(snac, actor, f_msg);
+        follower_add(snac, actor);
 
         snac_log(snac, xs_fmt("New follower %s", actor));
         do_notify = 1;
@@ -918,6 +917,7 @@ int process_message(snac *snac, char *msg, char *req)
     if (strcmp(type, "Update") == 0) {
         if (strcmp(utype, "Person") == 0) {
             actor_add(snac, actor, xs_dict_get(msg, "object"));
+
             snac_log(snac, xs_fmt("updated actor %s", actor));
         }
         else
@@ -1017,10 +1017,10 @@ void process_queue(snac *snac)
             FILE *f;
             int ok = 0;
 
-            f = popen("/usr/sbin/sendmail -t", "w");
-            if (f) {
+            if ((f = popen("/usr/sbin/sendmail -t", "w")) != NULL) {
                 fprintf(f, "%s\n", msg);
-                if (pclose(f) != EOF) //this is a pipe stream not just a file
+
+                if (pclose(f) != -1)
                     ok = 1;
             }
 
@@ -1089,7 +1089,7 @@ int activitypub_get_handler(d_char *req, char *q_path,
     if (p_path == NULL) {
         /* if there was no component after the user, it's an actor request */
         msg = msg_actor(&snac);
-        *ctype = "application/ld+json";
+        *ctype = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"";
     }
     else
     if (strcmp(p_path, "outbox") == 0) {
diff --git a/data.c b/data.c
index 49f215b..51ee2c7 100644
--- a/data.c
+++ b/data.c
@@ -6,6 +6,7 @@
 #include "xs_json.h"
 #include "xs_openssl.h"
 #include "xs_glob.h"
+#include "xs_set.h"
 
 #include "snac.h"
 
@@ -14,12 +15,12 @@
 #include <sys/file.h>
 #include <fcntl.h>
 
-double db_layout = 2.1;
+double db_layout = 2.5;
 
 
 int db_upgrade(d_char **error);
 
-int srv_open(char *basedir)
+int srv_open(char *basedir, int auto_upgrade)
 /* opens a server */
 {
     int ret = 0;
@@ -69,7 +70,14 @@ int srv_open(char *basedir)
                     error = xs_fmt("DEBUG level set to %d from environment", dbglevel);
                 }
 
-                ret = db_upgrade(&error);
+                if (auto_upgrade)
+                    ret = db_upgrade(&error);
+                else {
+                    if (xs_number_get(xs_dict_get(srv_config, "layout")) < db_layout)
+                        error = xs_fmt("ERROR: disk layout changed - execute 'snac upgrade' first");
+                    else
+                        ret = 1;
+                }
             }
 
         }
@@ -178,24 +186,215 @@ d_char *user_list(void)
 }
 
 
-double mtime(char *fn)
-/* returns the mtime of a file or directory, or 0.0 */
+double mtime_nl(const char *fn, int *n_link)
+/* returns the mtime and number of links of a file or directory, or 0.0 */
 {
     struct stat st;
     double r = 0.0;
+    int n = 0;
 
-    if (fn && stat(fn, &st) != -1)
-        r = (double)st.st_mtim.tv_sec;
+    if (fn && stat(fn, &st) != -1) {
+        r = (double) st.st_mtim.tv_sec;
+        n = st.st_nlink;
+    }
+
+    if (n_link)
+        *n_link = n;
 
     return r;
 }
 
 
-/** object database 2.1+ **/
+/** database 2.1+ **/
+
+/** indexes **/
+
+int index_add_md5(const char *fn, const char *md5)
+/* adds an md5 to an index */
+{
+    int status = 201; /* Created */
+    FILE *f;
+
+    if ((f = fopen(fn, "a")) != NULL) {
+        flock(fileno(f), LOCK_EX);
+
+        /* ensure the position is at the end after getting the lock */
+        fseek(f, 0, SEEK_END);
+
+        fprintf(f, "%s\n", md5);
+        fclose(f);
+    }
+    else
+        status = 500;
+
+    return status;
+}
+
+
+int index_add(const char *fn, const char *id)
+/* adds an id to an index */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return index_add_md5(fn, md5);
+}
+
+
+int index_del(const char *fn, const char *md5)
+/* deletes an md5 from an index */
+{
+    int status = 404;
+    FILE *i, *o;
+
+    if ((i = fopen(fn, "r")) != NULL) {
+        flock(fileno(i), LOCK_EX);
+
+        xs *nfn = xs_fmt("%s.new", fn);
+        char line[256];
+
+        if ((o = fopen(nfn, "w")) != NULL) {
+            while (fgets(line, sizeof(line), i) != NULL) {
+                line[32] = '\0';
+                if (memcmp(line, md5, 32) != 0)
+                    fprintf(o, "%s\n", line);
+            }
+
+            fclose(o);
+
+            xs *ofn = xs_fmt("%s.bak", fn);
+
+            link(fn, ofn);
+            rename(nfn, fn);
+        }
+        else
+            status = 500;
+
+        fclose(i);
+    }
+    else
+        status = 500;
+
+    return status;
+}
+
+
+int index_in_md5(const char *fn, const char *md5)
+/* checks if the md5 is already in the index */
+{
+    FILE *f;
+    int ret = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+
+        while (!ret && fgets(line, sizeof(line), f) != NULL) {
+            line[32] = '\0';
+
+            if (strcmp(line, md5) == 0)
+                ret = 1;
+        }
+
+        fclose(f);
+    }
+
+    return ret;
+}
+
+
+int index_in(const char *fn, const char *id)
+/* checks if the object id is already in the index */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return index_in_md5(fn, md5);
+}
+
+
+int index_first(const char *fn, char *line, int size)
+/* reads the first entry of an index */
+{
+    FILE *f;
+    int ret = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        if (fgets(line, size, f) != NULL) {
+            line[32] = '\0';
+            ret = 1;
+        }
+
+        fclose(f);
+    }
+
+    return ret;
+}
+
+
+d_char *index_list(const char *fn, int max)
+/* returns an index as a list */
+{
+    d_char *list = NULL;
+    FILE *f;
+    int n = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+        list = xs_list_new();
+
+        while (n < max && fgets(line, sizeof(line), f) != NULL) {
+            line[32] = '\0';
+            list = xs_list_append(list, line);
+            n++;
+        }
+
+        fclose(f);
+    }
+
+    return list;
+}
+
+
+d_char *index_list_desc(const char *fn, int max)
+/* returns an index as a list, in reverse order */
+{
+    d_char *list = NULL;
+    FILE *f;
+    int n = 0;
+
+    if ((f = fopen(fn, "r")) != NULL) {
+        flock(fileno(f), LOCK_SH);
+
+        char line[256];
+        list = xs_list_new();
+
+        /* move to the end minus one entry */
+        if (!fseek(f, 0, SEEK_END) && !fseek(f, -33, SEEK_CUR)) {
+            while (n < max && fgets(line, sizeof(line), f) != NULL) {
+                line[32] = '\0';
+                list = xs_list_append(list, line);
+                n++;
+
+                /* move backwards 2 entries */
+                if (fseek(f, -66, SEEK_CUR) == -1)
+                    break;
+            }
+        }
+
+        fclose(f);
+    }
+
+    return list;
+}
+
+
+/** objects **/
 
 d_char *_object_fn_by_md5(const char *md5)
 {
-    xs *bfn = xs_fmt("%s/object/%c%c/", srv_basedir, md5[0], md5[1]);
+    xs *bfn = xs_fmt("%s/object/%c%c", srv_basedir, md5[0], md5[1]);
 
     mkdir(bfn, 0755);
 
@@ -203,19 +402,18 @@ d_char *_object_fn_by_md5(const char *md5)
 }
 
 
-d_char *_object_fn_by_id(const char *id)
+d_char *_object_fn(const char *id)
 {
     xs *md5 = xs_md5_hex(id, strlen(id));
-
     return _object_fn_by_md5(md5);
 }
 
 
-int object_get(const char *id, d_char **obj, const char *type)
-/* returns a loaded object, optionally of the requested type */
+int object_get_by_md5(const char *md5, d_char **obj, const char *type)
+/* returns a stored object, optionally of the requested type */
 {
     int status = 404;
-    xs *fn     = _object_fn_by_id(id);
+    xs *fn     = _object_fn_by_md5(md5);
     FILE *f;
 
     if ((f = fopen(fn, "r")) != NULL) {
@@ -247,13 +445,27 @@ int object_get(const char *id, d_char **obj, const char *type)
 }
 
 
-int object_add(const char *id, d_char *obj)
+int object_get(const char *id, d_char **obj, const char *type)
+/* returns a stored object, optionally of the requested type */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return object_get_by_md5(md5, obj, type);
+}
+
+
+int _object_add(const char *id, d_char *obj, int ow)
 /* stores an object */
 {
     int status = 201; /* Created */
-    xs *fn     = _object_fn_by_id(id);
+    xs *fn     = _object_fn(id);
     FILE *f;
 
+    if (!ow && mtime(fn) > 0.0) {
+        /* object already here */
+        srv_debug(0, xs_fmt("object_add object already here %s", id));
+        return 204; /* No content */
+    }
+
     if ((f = fopen(fn, "w")) != NULL) {
         flock(fileno(f), LOCK_EX);
 
@@ -261,100 +473,241 @@ int object_add(const char *id, d_char *obj)
 
         fwrite(j, strlen(j), 1, f);
         fclose(f);
+
+        /* does this object has a parent? */
+        char *in_reply_to = xs_dict_get(obj, "inReplyTo");
+
+        if (!xs_is_null(in_reply_to) && *in_reply_to) {
+            /* update the children index of the parent */
+            xs *c_idx = _object_fn(in_reply_to);
+
+            c_idx = xs_replace_i(c_idx, ".json", "_c.idx");
+            index_add(c_idx, id);
+
+            srv_debug(0, xs_fmt("object_add added child %s to %s", id, c_idx));
+
+            /* create a one-element index with the parent */
+            xs *p_idx = xs_replace(fn, ".json", "_p.idx");
+            index_add(p_idx, in_reply_to);
+
+            srv_debug(0, xs_fmt("object_add added parent %s to %s", in_reply_to, p_idx));
+        }
     }
     else
         status = 500;
 
+    srv_debug(0, xs_fmt("object_add %s %s %d", id, fn, status));
+
     return status;
 }
 
 
-d_char *_follower_fn(snac *snac, char *actor)
+int object_add(const char *id, d_char *obj)
+/* stores an object */
 {
-    xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/followers/%s.json", snac->basedir, md5);
+    return _object_add(id, obj, 0);
 }
 
 
-int follower_add(snac *snac, char *actor, char *msg)
-/* adds a follower */
+int object_add_ow(const char *id, d_char *obj)
+/* stores an object (overwriting allowed) */
 {
-    int ret = 201; /* created */
-    xs *fn = _follower_fn(snac, actor);
-    FILE *f;
+    return _object_add(id, obj, 1);
+}
 
-    if ((f = fopen(fn, "w")) != NULL) {
-        xs *j = xs_json_dumps_pp(msg, 4);
 
-        fwrite(j, 1, strlen(j), f);
-        fclose(f);
+int object_del_by_md5(const char *md5)
+/* deletes an object by its md5 */
+{
+    int status = 404;
+    xs *fn     = _object_fn_by_md5(md5);
+
+    if (fn != NULL && unlink(fn) != -1) {
+        status = 200;
+
+        /* also delete associated indexes */
+        xs *spec  = xs_dup(fn);
+        spec      = xs_replace_i(spec, ".json", "*.idx");
+        xs *files = xs_glob(spec, 0, 0);
+        char *p, *v;
+
+        p = files;
+        while (xs_list_iter(&p, &v)) {
+            srv_debug(0, xs_fmt("object_del index %s", v));
+            unlink(v);
+        }
     }
-    else
-        ret = 500;
 
-    snac_debug(snac, 2, xs_fmt("follower_add %s %s", actor, fn));
+    srv_debug(0, xs_fmt("object_del %s %d", fn, status));
+
+    return status;
+}
+
+
+int object_del(const char *id)
+/* deletes an object */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return object_del_by_md5(md5);
+}
+
+
+int object_del_if_unref(const char *id)
+/* deletes an object if its n_links < 2 */
+{
+    xs *fn = _object_fn(id);
+    int n_links;
+    int ret = 0;
+
+    if (mtime_nl(fn, &n_links) > 0.0 && n_links < 2)
+        ret = object_del(id);
 
     return ret;
 }
 
 
-int follower_del(snac *snac, char *actor)
-/* deletes a follower */
+d_char *object_children(const char *id)
+/* returns the list of an object's children */
+{
+    xs *fn = _object_fn(id);
+
+    fn = xs_replace_i(fn, ".json", "_c.idx");
+
+    return index_list(fn, XS_ALL);
+}
+
+
+int object_admire(const char *id, const char *actor, int like)
+/* actor likes or announces this object */
 {
     int status = 200;
-    xs *fn = _follower_fn(snac, actor);
+    xs *fn     = _object_fn(id);
 
-    if (fn != NULL)
-        unlink(fn);
-    else
-        status = 404;
+    fn = xs_replace_i(fn, ".json", like ? "_l.idx" : "_a.idx");
 
-    snac_debug(snac, 2, xs_fmt("follower_del %s %s", actor, fn));
+    if (!index_in(fn, actor)) {
+        status = index_add(fn, actor);
+
+        srv_debug(0, xs_fmt("object_admire (%s) %s %s", like ? "Like" : "Announce", actor, fn));
+    }
 
     return status;
 }
 
 
-int follower_check(snac *snac, char *actor)
-/* checks if someone is a follower */
+int _object_user_cache(snac *snac, const char *id, const char *cachedir, int del)
+/* adds or deletes from a user cache */
 {
-    xs *fn = _follower_fn(snac, actor);
+    xs *ofn = _object_fn(id);
+    xs *l   = xs_split(ofn, "/");
+    xs *cfn = xs_fmt("%s/%s/%s", snac->basedir, cachedir, xs_list_get(l, -1));
+    xs *idx = xs_fmt("%s/%s.idx", snac->basedir, cachedir);
+    int ret;
+
+    if (del) {
+        if ((ret = unlink(cfn)) != -1)
+            index_del(idx, id);
+    }
+    else {
+        index_add(idx, id);
+        ret = link(ofn, cfn);
+    }
 
-    return !!(mtime(fn) != 0.0);
+    return ret;
+}
+
+
+int object_user_cache_add(snac *snac, const char *id, const char *cachedir)
+/* caches an object into a user cache */
+{
+    return _object_user_cache(snac, id, cachedir, 0);
+}
+
+
+int object_user_cache_del(snac *snac, const char *id, const char *cachedir)
+/* deletes an object from a user cache */
+{
+    return _object_user_cache(snac, id, cachedir, 1);
+}
+
+
+int object_user_cache_in(snac *snac, const char *id, const char *cachedir)
+/* checks if an object is stored in a cache */
+{
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    xs *cfn = xs_fmt("%s/%s/%s.json", snac->basedir, cachedir, md5);
+
+    return !!(mtime(cfn) != 0.0);
+}
+
+
+d_char *object_user_cache_list(snac *snac, const char *cachedir, int max)
+/* returns the objects in a cache as a list */
+{
+    xs *idx = xs_fmt("%s/%s.idx", snac->basedir, cachedir);
+    return index_list(idx, max);
+}
+
+
+/** specialized functions **/
+
+/** followers **/
+
+int follower_add(snac *snac, const char *actor)
+/* adds a follower */
+{
+    int ret = object_user_cache_add(snac, actor, "followers");
+
+    snac_debug(snac, 2, xs_fmt("follower_add %s %s", actor));
+
+    return ret == -1 ? 500 : 200;
+}
+
+
+int follower_del(snac *snac, const char *actor)
+/* deletes a follower */
+{
+    int ret = object_user_cache_del(snac, actor, "followers");
+
+    snac_debug(snac, 2, xs_fmt("follower_del %s %s", actor));
+
+    return ret == -1 ? 404 : 200;
+}
+
+
+int follower_check(snac *snac, const char *actor)
+/* checks if someone is a follower */
+{
+    return object_user_cache_in(snac, actor, "followers");
 }
 
 
 d_char *follower_list(snac *snac)
 /* returns the list of followers */
 {
-    xs *spec = xs_fmt("%s/followers/" "*.json", snac->basedir);
-    xs *glist = xs_glob(spec, 0, 0);
+    xs *list      = object_user_cache_list(snac, "followers", XS_ALL);
+    d_char *fwers = xs_list_new();
     char *p, *v;
-    d_char *list = xs_list_new();
 
-    /* iterate the list of files */
-    p = glist;
+    /* resolve the list of md5 to be a list of actors */
+    p = list;
     while (xs_list_iter(&p, &v)) {
-        FILE *f;
-
-        /* load the follower data */
-        if ((f = fopen(v, "r")) != NULL) {
-            xs *j = xs_readall(f);
-            fclose(f);
+        xs *a_obj = NULL;
 
-            if (j != NULL) {
-                xs *o = xs_json_loads(j);
+        if (valid_status(object_get_by_md5(v, &a_obj, NULL))) {
+            char *actor = xs_dict_get(a_obj, "id");
 
-                if (o != NULL)
-                    list = xs_list_append(list, o);
-            }
+            if (!xs_is_null(actor))
+                fwers = xs_list_append(fwers, actor);
         }
     }
 
-    return list;
+    return fwers;
 }
 
 
+/** timeline **/
+
 double timeline_mtime(snac *snac)
 {
     xs *fn = xs_fmt("%s/timeline", snac->basedir);
@@ -437,6 +790,13 @@ int timeline_del(snac *snac, char *id)
         ret = 200;
     }
 
+    /* delete from the user's caches */
+    object_user_cache_del(snac, id, "public");
+    object_user_cache_del(snac, id, "private");
+
+    /* try to delete the object if it's not used elsewhere */
+    object_del_if_unref(id);
+
     return ret;
 }
 
@@ -511,22 +871,11 @@ int _timeline_write(snac *snac, char *id, char *msg, char *parent, char *referre
 
         if (pfn != NULL && (f = fopen(pfn, "r")) != NULL) {
             xs *j;
-            char *v;
 
             j = xs_readall(f);
             fclose(f);
 
             p_msg = xs_json_loads(j);
-
-            if ((v = xs_dict_get(p_msg, "_snac")) != NULL) {
-                /* is parent hidden? */
-                if ((v = xs_dict_get(v, "hidden")) && xs_type(v) == XSTYPE_TRUE) {
-                    snac_debug(snac, 1,
-                        xs_fmt("_timeline_write dropping due to hidden parent %s (%s)", id, parent));
-
-                    return 0;
-                }
-            }
         }
     }
 
@@ -648,6 +997,16 @@ int _timeline_write(snac *snac, char *id, char *msg, char *parent, char *referre
 }
 
 
+void timeline_update_indexes(snac *snac, const char *id)
+/* updates the indexes */
+{
+    object_user_cache_add(snac, id, "private");
+
+    if (xs_startswith(id, snac->actor))
+        object_user_cache_add(snac, id, "public");
+}
+
+
 int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer)
 /* adds a message to the timeline */
 {
@@ -680,13 +1039,61 @@ int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer
 
     msg = xs_dict_set(msg, "_snac", md);
 
-    if ((ret = _timeline_write(snac, id, msg, parent, referrer)))
+    if ((ret = _timeline_write(snac, id, msg, parent, referrer))) {
         snac_debug(snac, 1, xs_fmt("timeline_add %s", id));
 
+        object_add(id, o_msg);
+        timeline_update_indexes(snac, id);
+    }
+
     return ret;
 }
 
 
+d_char *timeline_top_level(snac *snac, d_char *list)
+/* returns the top level md5 entries from this index */
+{
+    d_char *tl = xs_list_new();
+    xs_set seen;
+    char *p, *v;
+
+    xs_set_init(&seen);
+
+    p = list;
+    while (xs_list_iter(&p, &v)) {
+        char line[256] = "";
+
+        strcpy(line, v);
+
+        for (;;) {
+            char line2[256];
+            xs *fn = _object_fn_by_md5(line);
+            fn     = xs_replace_i(fn, ".json", "_p.idx");
+
+            /* if it doesn't have a parent, use this */
+            if (index_first(fn, line2, sizeof(line2)) == 0)
+                break;
+
+            xs *pfn = _object_fn_by_md5(line2);
+
+            /* well, there is a parent... but if it's not there, use this */
+            if (mtime(pfn) == 0.0)
+                break;
+
+            /* it's here! try again with its own parent */
+            strcpy(line, line2);
+        }
+
+        if (xs_set_add(&seen, line) == 1)
+            tl = xs_list_append(tl, line);
+    }
+
+    xs_set_free(&seen);
+
+    return tl;
+}
+
+
 void timeline_admire(snac *snac, char *id, char *admirer, int like)
 /* updates a timeline entry with a new admiration */
 {
@@ -734,52 +1141,16 @@ void timeline_admire(snac *snac, char *id, char *admirer, int like)
     }
     else
         snac_log(snac, xs_fmt("timeline_admire ignored for unknown object %s", id));
-}
-
-
-int timeline_hide(snac *snac, char *id, int hide)
-/* hides/unhides a timeline entry */
-{
-    int ret = 0;
-    xs *fn = _timeline_find_fn(snac, id);
-    FILE *f;
-
-    if (fn != NULL && (f = fopen(fn, "r")) != NULL) {
-        xs *s1   = xs_readall(f);
-        xs *msg  = xs_json_loads(s1);
-        xs *meta = xs_dup(xs_dict_get(msg, "_snac"));
-        xs *hdn  = xs_val_new(hide ? XSTYPE_TRUE : XSTYPE_FALSE);
-        char *p, *v;
-
-        fclose(f);
-
-        /* if it's already in this hidden state, we're done */
-        if ((v = xs_dict_get(meta, "hidden")) && xs_type(v) == xs_type(hdn))
-            return ret;
-
-        meta = xs_dict_set(meta, "hidden", hdn);
-        msg  = xs_dict_set(msg,  "_snac",  meta);
-
-        if ((f = fopen(fn, "w")) != NULL) {
-            xs *j1 = xs_json_dumps_pp(msg, 4);
-
-            fwrite(j1, strlen(j1), 1, f);
-            fclose(f);
 
-            snac_debug(snac, 1, xs_fmt("timeline_hide %d %s", hide, id));
+    object_admire(id, admirer, like);
+}
 
-            /* now hide the children */
-            p = xs_dict_get(meta, "children");
-            while (xs_list_iter(&p, &v))
-                timeline_hide(snac, v, hide);
 
-            ret = 1;
-        }
-    }
-
-    return ret;
-}
+/** following **/
 
+/* this needs special treatment and cannot use the object db as is,
+   with a link to a cached author, because we need the Follow object
+   in case we need to unfollow (Undo + original Follow) */
 
 d_char *_following_fn(snac *snac, char *actor)
 {
@@ -811,7 +1182,7 @@ int following_add(snac *snac, char *actor, char *msg)
 
 
 int following_del(snac *snac, char *actor)
-/* someone is no longer following us */
+/* we're not following this actor any longer */
 {
     xs *fn = _following_fn(snac, actor);
 
@@ -824,7 +1195,7 @@ int following_del(snac *snac, char *actor)
 
 
 int following_check(snac *snac, char *actor)
-/* checks if someone is following us */
+/* checks if we are following this actor */
 {
     xs *fn = _following_fn(snac, actor);
 
@@ -877,8 +1248,12 @@ d_char *following_list(snac *snac)
                 if (o != NULL) {
                     char *type = xs_dict_get(o, "type");
 
-                    if (!xs_is_null(type) && strcmp(type, "Accept") == 0)
-                        list = xs_list_append(list, o);
+                    if (!xs_is_null(type) && strcmp(type, "Accept") == 0) {
+                        char *actor = xs_dict_get(o, "actor");
+
+                        if (!xs_is_null(actor))
+                            list = xs_list_append(list, actor);
+                    }
                 }
             }
         }
@@ -891,7 +1266,7 @@ d_char *following_list(snac *snac)
 d_char *_muted_fn(snac *snac, char *actor)
 {
     xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/muted/%s.json", snac->basedir, md5);
+    return xs_fmt("%s/muted/%s", snac->basedir, md5);
 }
 
 
@@ -930,65 +1305,89 @@ int is_muted(snac *snac, char *actor)
 }
 
 
-d_char *_actor_fn(snac *snac, char *actor)
-/* returns the file name for an actor */
+d_char *_hidden_fn(snac *snac, const char *id)
 {
-    xs *md5 = xs_md5_hex(actor, strlen(actor));
-    return xs_fmt("%s/actors/%s.json", snac->basedir, md5);
+    xs *md5 = xs_md5_hex(id, strlen(id));
+    return xs_fmt("%s/hidden/%s", snac->basedir, md5);
 }
 
 
-int actor_add(snac *snac, char *actor, char *msg)
-/* adds an actor */
+void hide(snac *snac, const char *id)
+/* hides a message tree */
 {
-    int ret = 201; /* created */
-    xs *fn = _actor_fn(snac, actor);
+    xs *fn = _hidden_fn(snac, id);
     FILE *f;
 
     if ((f = fopen(fn, "w")) != NULL) {
-        xs *j = xs_json_dumps_pp(msg, 4);
-
-        fwrite(j, 1, strlen(j), f);
+        fprintf(f, "%s\n", id);
         fclose(f);
+
+        snac_debug(snac, 2, xs_fmt("hidden %s %s", id, fn));
+
+        /* hide all the children */
+        xs *chld = object_children(id);
+        char *p, *v;
+
+        p = chld;
+        while (xs_list_iter(&p, &v)) {
+            xs *co = NULL;
+
+            /* resolve to get the id */
+            if (valid_status(object_get_by_md5(v, &co, NULL))) {
+                if ((v = xs_dict_get(co, "id")) != NULL)
+                    hide(snac, v);
+            }
+        }
     }
-    else
-        ret = 500;
+}
 
-    snac_debug(snac, 2, xs_fmt("actor_add %s %s", actor, fn));
 
-//    object_add(actor, msg);
+int is_hidden(snac *snac, const char *id)
+/* check is id is hidden */
+{
+    xs *fn = _hidden_fn(snac, id);
 
-    return ret;
+    return !!(mtime(fn) != 0.0);
+}
+
+
+int actor_add(snac *snac, const char *actor, d_char *msg)
+/* adds an actor */
+{
+    return object_add_ow(actor, msg);
 }
 
 
-int actor_get(snac *snac, char *actor, d_char **data)
+int actor_get(snac *snac, const char *actor, d_char **data)
 /* returns an already downloaded actor */
 {
-    xs *fn = _actor_fn(snac, actor);
-    double t;
-    double max_time;
-    int status;
-    FILE *f;
+    int status = 200;
+    char *d;
 
     if (strcmp(actor, snac->actor) == 0) {
+        /* this actor */
         if (data)
             *data = msg_actor(snac);
 
-        return 200;
+        return status;
     }
 
-    t = mtime(fn);
+    /* read the object */
+    if (!valid_status(status = object_get(actor, &d, NULL)))
+        return status;
 
-    /* no mtime? there is nothing here */
-    if (t == 0.0)
-        return 404;
+    if (data)
+        *data = d;
+
+    xs *fn = _object_fn(actor);
+    double max_time;
 
     /* maximum time for the actor data to be considered stale */
     max_time = 3600.0 * 36.0;
 
-    if (t + max_time < (double) time(NULL)) {
+    if (mtime(fn) + max_time < (double) time(NULL)) {
         /* actor data exists but also stinks */
+        FILE *f;
 
         if ((f = fopen(fn, "a")) != NULL) {
             /* write a blank at the end to 'touch' the file */
@@ -998,22 +1397,6 @@ int actor_get(snac *snac, char *actor, d_char **data)
 
         status = 205; /* "205: Reset Content" "110: Response Is Stale" */
     }
-    else {
-        /* it's still valid */
-        status = 200;
-    }
-
-    if (data) {
-        if ((f = fopen(fn, "r")) != NULL) {
-            xs *j = xs_readall(f);
-
-            fclose(f);
-
-            *data = xs_json_loads(j);
-        }
-        else
-            status = 500;
-    }
 
     return status;
 }
@@ -1033,7 +1416,7 @@ int static_get(snac *snac, const char *id, d_char **data, int *size)
     FILE *f;
     int status = 404;
 
-    *size = 0xfffffff;
+    *size = XS_ALL;
 
     if ((f = fopen(fn, "rb")) != NULL) {
         *data = xs_read(f, size);
@@ -1121,6 +1504,8 @@ d_char *history_list(snac *snac)
 }
 
 
+/** the queue **/
+
 static int _enqueue_put(char *fn, char *msg)
 /* writes safely to the queue */
 {
@@ -1267,38 +1652,81 @@ d_char *dequeue(snac *snac, char *fn)
 }
 
 
+/** the purge **/
+
+static void _purge_file(const char *fn, time_t mt)
+/* purge fn if it's older than days */
+{
+    if (mtime(fn) < mt) {
+        /* older than the minimum time: delete it */
+        unlink(fn);
+        srv_debug(1, xs_fmt("purged %s", fn));
+    }
+}
+
+
 static void _purge_subdir(snac *snac, const char *subdir, int days)
 /* purges all files in subdir older than days */
 {
     if (days) {
         time_t mt = time(NULL) - days * 24 * 3600;
-        xs *spec  = xs_fmt("%s/%s/" "*.json", snac->basedir, subdir);
+        xs *spec  = xs_fmt("%s/%s/" "*", snac->basedir, subdir);
         xs *list  = xs_glob(spec, 0, 0);
         char *p, *v;
 
         p = list;
-        while (xs_list_iter(&p, &v)) {
-            if (mtime(v) < mt) {
-                /* older than the minimum time: delete it */
-                unlink(v);
-                snac_debug(snac, 1, xs_fmt("purged %s", v));
+        while (xs_list_iter(&p, &v))
+            _purge_file(v, mt);
+    }
+}
+
+
+void purge_server(void)
+/* purge global server data */
+{
+    int tpd = xs_number_get(xs_dict_get(srv_config, "timeline_purge_days"));
+    xs *spec = xs_fmt("%s/object/??", srv_basedir);
+    xs *dirs = xs_glob(spec, 0, 0);
+    char *p, *v;
+
+    time_t mt = time(NULL) - tpd * 24 * 3600;
+
+    p = dirs;
+    while (xs_list_iter(&p, &v)) {
+        xs *spec2 = xs_fmt("%s/" "*.json", v);
+        xs *files = xs_glob(spec2, 0, 0);
+        char *p2, *v2;
+
+        p2 = files;
+        while (xs_list_iter(&p2, &v2)) {
+            int n_link;
+
+            /* old and with no hard links? */
+            if (mtime_nl(v2, &n_link) < mt && n_link < 2) {
+                xs *s1    = xs_replace(v2, ".json", "");
+                xs *l     = xs_split(s1, "/");
+                char *md5 = xs_list_get(l, -1);
+
+                object_del_by_md5(md5);
             }
         }
     }
 }
 
 
-void purge(snac *snac)
-/* do the purge */
+void purge_user(snac *snac)
+/* do the purge for this user */
 {
     int days;
 
     days = xs_number_get(xs_dict_get(srv_config, "timeline_purge_days"));
     _purge_subdir(snac, "timeline", days);
-    _purge_subdir(snac, "actors", days);
+    _purge_subdir(snac, "hidden", days);
+    _purge_subdir(snac, "private", days);
 
     days = xs_number_get(xs_dict_get(srv_config, "local_purge_days"));
     _purge_subdir(snac, "local", days);
+    _purge_subdir(snac, "public", days);
 }
 
 
@@ -1312,8 +1740,10 @@ void purge_all(void)
     p = list;
     while (xs_list_iter(&p, &uid)) {
         if (user_open(&snac, uid)) {
-            purge(&snac);
+            purge_user(&snac);
             user_free(&snac);
         }
     }
+
+    purge_server();
 }
diff --git a/doc/snac.1 b/doc/snac.1
index 5303bc8..3e34725 100644
--- a/doc/snac.1
+++ b/doc/snac.1
@@ -145,6 +145,11 @@ Initializes the database. This is an interactive command; necessary
 information will be prompted for. The
 .Ar basedir
 directory must not exist.
+.It Cm upgrade Ar basedir
+Upgrades the database disk layout after installing a new version.
+Only necessary if
+.Nm
+complains and demands it.
 .It Cm purge Ar basedir
 Purges old data from the timeline of all users.
 .It Cm adduser Ar basedir Op uid
diff --git a/doc/snac.8 b/doc/snac.8
index a241158..60671a1 100644
--- a/doc/snac.8
+++ b/doc/snac.8
@@ -14,6 +14,19 @@ This is the admin manual. For user operation, see
 .Xr snac 1 .
 For file and data formats, see
 .Xr snac 5 .
+.Ss Special cares about your snac you must know beforehand
+.Nm
+makes heavy use of hard links and link reference counts for its work, so
+don't even think of using it on a filesystem that doesn't support this
+feature. Most UNIX-like operating systems (Linux, the BSDs, the old DEC
+Ultrix machine in your grandfather basement, probably MacOS) support hard
+links on their native filesystems. Don't do fancy things like moving the
+subdirectories to different filesystems. Also, if you move your
+.Nm
+installation to another server, do it with a tool that respect hard
+link counts. Remember:
+.Nm
+is a very UNIXy program that loves hard links.
 .Ss Building and Installation
 A C compiler must be installed in the system, as well as the development
 headers and libraries for OpenSSL and curl. To build
@@ -69,6 +82,19 @@ startup scripts and configuration data in the
 directory.
 For other operating systems, please read the appropriate documentation
 on how to install a daemon as a non-root service.
+.Ss Upgrading to a new version
+Sometimes, the database disk layout changes between versions. If there
+is such a change,
+.Nm
+will refuse to run and require an upgrade. Do this by running
+.Bd -literal -offset indent
+snac upgrade $HOME/snac-data
+.Ed
+Take special care to execute this upgrade operation without any
+.Nm
+processes serving on the same folder. You can break everything. I know
+this because Tyler knows this.
+.Pp
 .Ss Server Setup
 .Pp
 An http server with TLS and proxying support must already be
diff --git a/html.c b/html.c
index 3f59b09..63b54c3 100644
--- a/html.c
+++ b/html.c
@@ -196,14 +196,14 @@ d_char *html_user_header(snac *snac, d_char *s, int local)
                 "<a href=\"%s.rss\">%s</a> - "
                 "<a href=\"%s/admin\" rel=\"nofollow\">%s</a></nav>\n",
                 snac->actor, L("RSS"),
-                snac->actor, L("admin"));
+                snac->actor, L("private"));
         else
             s1 = xs_fmt(
                 "<a href=\"%s\">%s</a> - "
                 "<a href=\"%s/admin\">%s</a> - "
                 "<a href=\"%s/people\">%s</a></nav>\n",
                 snac->actor, L("public"),
-                snac->actor, L("admin"),
+                snac->actor, L("private"),
                 snac->actor, L("people"));
 
         s = xs_str_cat(s, s1);
@@ -497,7 +497,7 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
     xs *s = xs_str_new(NULL);
 
     /* top wrap */
-    if ((v = xs_dict_get(meta, "hidden")) && xs_type(v) == XSTYPE_TRUE)
+    if (is_hidden(snac, id))
         s = xs_str_cat(s, "<div style=\"display: none\">\n");
     else
         s = xs_str_cat(s, "<div>\n");
@@ -840,13 +840,12 @@ d_char *html_people_list(snac *snac, d_char *os, d_char *list, const char *heade
 {
     xs *s = xs_str_new(NULL);
     xs *h = xs_fmt("<h2>%s</h2>\n", header);
-    char *p, *v;
+    char *p, *actor_id;
 
     s = xs_str_cat(s, h);
 
     p = list;
-    while (xs_list_iter(&p, &v)) {
-        char *actor_id = xs_dict_get(v, "actor");
+    while (xs_list_iter(&p, &actor_id)) {
         xs *md5 = xs_md5_hex(actor_id, strlen(actor_id));
         xs *actor = NULL;
 
@@ -1008,7 +1007,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char *
             status  = 200;
         }
         else {
-            xs *list = local_list(&snac, 0xfffffff);
+            xs *list = local_list(&snac, XS_ALL);
 
             *body   = html_timeline(&snac, list, 1);
             *b_size = strlen(*body);
@@ -1034,7 +1033,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char *
             else {
                 snac_debug(&snac, 1, xs_fmt("building timeline"));
 
-                xs *list = timeline_list(&snac, 0xfffffff);
+                xs *list = timeline_list(&snac, XS_ALL);
 
                 *body   = html_timeline(&snac, list, 0);
                 *b_size = strlen(*body);
@@ -1300,7 +1299,7 @@ int html_post_handler(d_char *req, char *q_path, d_char *payload, int p_size,
         }
         else
         if (strcmp(action, L("Hide")) == 0) {
-            timeline_hide(&snac, id, 1);
+            hide(&snac, id);
         }
         else
         if (strcmp(action, L("Follow")) == 0) {
@@ -1341,6 +1340,9 @@ int html_post_handler(d_char *req, char *q_path, d_char *payload, int p_size,
 
                 post(&snac, msg);
 
+                /* FIXME: also post this Tombstone to people
+                   that Announce'd it */
+
                 snac_log(&snac, xs_fmt("posted tombstone for %s", id));
             }
 
diff --git a/httpd.c b/httpd.c
index 76bebad..dd8978d 100644
--- a/httpd.c
+++ b/httpd.c
@@ -222,7 +222,7 @@ static void *queue_thread(void *arg)
     time_t purge_time;
 
     /* first purge time */
-    purge_time = time(NULL) + 15 * 60;
+    purge_time = time(NULL) + 10 * 60;
 
     srv_log(xs_fmt("queue thread start"));
 
diff --git a/main.c b/main.c
index 8e3dbd2..a109192 100644
--- a/main.c
+++ b/main.c
@@ -16,6 +16,7 @@ int usage(void)
     printf("Commands:\n");
     printf("\n");
     printf("init [{basedir}]                 Initializes the database\n");
+    printf("upgrade {basedir}                Upgrade to a new version\n");
     printf("adduser {basedir} [{uid}]        Adds a new user\n");
     printf("httpd {basedir}                  Starts the HTTPD daemon\n");
     printf("purge {basedir}                  Purges old data\n");
@@ -76,6 +77,19 @@ int main(int argc, char *argv[])
         return initdb(basedir);
     }
 
+    if (strcmp(cmd, "upgrade") == 0) {
+        int ret;
+
+        /* database upgrade */
+        if ((basedir = GET_ARGV()) == NULL)
+            return usage();
+
+        if ((ret = srv_open(basedir, 1)) == 1)
+            srv_log(xs_dup("OK"));
+
+        return ret;
+    }
+
     if (strcmp(cmd, "markdown") == 0) {
         /* undocumented, for testing only */
         xs *c = xs_readall(stdin);
@@ -88,7 +102,7 @@ int main(int argc, char *argv[])
     if ((basedir = GET_ARGV()) == NULL)
         return usage();
 
-    if (!srv_open(basedir)) {
+    if (!srv_open(basedir, 0)) {
         srv_log(xs_fmt("error opening database at %s", basedir));
         return 1;
     }
@@ -142,12 +156,21 @@ int main(int argc, char *argv[])
     }
 
     if (strcmp(cmd, "timeline") == 0) {
-        xs *list = local_list(&snac, 0xfffffff);
+#if 0
+        xs *list = local_list(&snac, XS_ALL);
         xs *body = html_timeline(&snac, list, 1);
 
         printf("%s\n", body);
         user_free(&snac);
         srv_free();
+#endif
+
+        xs *idx  = xs_fmt("%s/private.idx", snac.basedir);
+        xs *list = index_list_desc(idx, 256);
+        xs *tl   = timeline_top_level(&snac, list);
+
+        xs *j    = xs_json_dumps_pp(tl, 4);
+        printf("%s\n", j);
 
         return 0;
     }
diff --git a/snac.h b/snac.h
index 5427497..b8b5215 100644
--- a/snac.h
+++ b/snac.h
@@ -24,7 +24,7 @@ double ftime(void);
 void srv_debug(int level, d_char *str);
 #define srv_log(str) srv_debug(0, str)
 
-int srv_open(char *basedir);
+int srv_open(char *basedir, int auto_upgrade);
 void srv_free(void);
 
 typedef struct _snac {
@@ -50,11 +50,21 @@ int check_password(char *uid, char *passwd, char *hash);
 void srv_archive(char *direction, char *req, char *payload, int p_size,
                  int status, char *headers, char *body, int b_size);
 
-double mtime(char *fn);
+double mtime_nl(const char *fn, int *n_link);
+#define mtime(fn) mtime_nl(fn, NULL)
 
-int follower_add(snac *snac, char *actor, char *msg);
-int follower_del(snac *snac, char *actor);
-int follower_check(snac *snac, char *actor);
+int index_add(const char *fn, const char *md5);
+int index_del(const char *fn, const char *md5);
+int index_first(const char *fn, char *buf, int size);
+d_char *index_list(const char *fn, int max);
+d_char *index_list_desc(const char *fn, int max);
+
+int object_del(const char *id);
+int object_del_if_unref(const char *id);
+
+int follower_add(snac *snac, const char *actor);
+int follower_del(snac *snac, const char *actor);
+int follower_check(snac *snac, const char *actor);
 d_char *follower_list(snac *snac);
 
 double timeline_mtime(snac *snac);
@@ -66,7 +76,8 @@ d_char *timeline_get(snac *snac, char *fn);
 d_char *timeline_list(snac *snac, int max);
 int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer);
 void timeline_admire(snac *snac, char *id, char *admirer, int like);
-int timeline_hide(snac *snac, char *id, int hide);
+
+d_char *timeline_top_level(snac *snac, d_char *list);
 
 d_char *local_list(snac *snac, int max);
 
@@ -80,8 +91,11 @@ void mute(snac *snac, char *actor);
 void unmute(snac *snac, char *actor);
 int is_muted(snac *snac, char *actor);
 
-int actor_add(snac *snac, char *actor, char *msg);
-int actor_get(snac *snac, char *actor, d_char **data);
+void hide(snac *snac, const char *id);
+int is_hidden(snac *snac, const char *id);
+
+int actor_add(snac *snac, const char *actor, d_char *msg);
+int actor_get(snac *snac, const char *actor, d_char **data);
 
 int static_get(snac *snac, const char *id, d_char **data, int *size);
 void static_put(snac *snac, const char *id, const char *data, int size);
diff --git a/upgrade.c b/upgrade.c
index e4c75bb..e2983e4 100644
--- a/upgrade.c
+++ b/upgrade.c
@@ -40,6 +40,137 @@ int db_upgrade(d_char **error)
 
             nf = 2.1;
         }
+        else
+        if (f < 2.2) {
+            xs *users = user_list();
+            char *p, *v;
+
+            p = users;
+            while (xs_list_iter(&p, &v)) {
+                snac snac;
+
+                if (user_open(&snac, v)) {
+                    xs *spec = xs_fmt("%s/actors/" "*.json", snac.basedir);
+                    xs *list = xs_glob(spec, 0, 0);
+                    char *g, *fn;
+
+                    g = list;
+                    while (xs_list_iter(&g, &fn)) {
+                        xs *l   = xs_split(fn, "/");
+                        char *b = xs_list_get(l, -1);
+                        xs *dir = xs_fmt("%s/object/%c%c", srv_basedir, b[0], b[1]);
+                        xs *nfn = xs_fmt("%s/%s", dir, b);
+
+                        mkdir(dir, 0755);
+                        rename(fn, nfn);
+                    }
+
+                    xs *odir = xs_fmt("%s/actors", snac.basedir);
+                    rmdir(odir);
+
+                    user_free(&snac);
+                }
+            }
+
+            nf = 2.2;
+        }
+        else
+        if (f < 2.3) {
+            xs *users = user_list();
+            char *p, *v;
+
+            p = users;
+            while (xs_list_iter(&p, &v)) {
+                snac snac;
+
+                if (user_open(&snac, v)) {
+                    char *p, *v;
+                    xs *dir = xs_fmt("%s/hidden", snac.basedir);
+
+                    /* create the hidden directory */
+                    mkdir(dir, 0755);
+
+                    /* rename all muted files incorrectly named .json */
+                    xs *spec = xs_fmt("%s/muted/" "*.json", snac.basedir);
+                    xs *fns  = xs_glob(spec, 0, 0);
+
+                    p = fns;
+                    while (xs_list_iter(&p, &v)) {
+                        xs *nfn = xs_replace(v, ".json", "");
+                        rename(v, nfn);
+                    }
+
+                    user_free(&snac);
+                }
+            }
+
+            nf = 2.3;
+        }
+        else
+        if (f < 2.4) {
+            xs *users = user_list();
+            char *p, *v;
+
+            p = users;
+            while (xs_list_iter(&p, &v)) {
+                snac snac;
+
+                if (user_open(&snac, v)) {
+                    xs *dir = xs_fmt("%s/public", snac.basedir);
+                    mkdir(dir, 0755);
+
+                    dir = xs_replace_i(dir, "public", "private");
+                    mkdir(dir, 0755);
+
+                    user_free(&snac);
+                }
+            }
+
+            nf = 2.4;
+        }
+        else
+        if (f < 2.5) {
+            /* upgrade followers */
+            xs *users = user_list();
+            char *p, *v;
+
+            p = users;
+            while (xs_list_iter(&p, &v)) {
+                snac snac;
+
+                if (user_open(&snac, v)) {
+                    xs *spec = xs_fmt("%s/followers/" "*.json", snac.basedir);
+                    xs *dir  = xs_glob(spec, 0, 0);
+                    char *p, *v;
+
+                    p = dir;
+                    while (xs_list_iter(&p, &v)) {
+                        FILE *f;
+
+                        if ((f = fopen(v, "r")) != NULL) {
+                            xs *s = xs_readall(f);
+                            xs *o = xs_json_loads(s);
+                            fclose(f);
+
+                            char *type = xs_dict_get(o, "type");
+
+                            if (!xs_is_null(type) && strcmp(type, "Follow") == 0) {
+                                unlink(v);
+
+                                char *actor = xs_dict_get(o, "actor");
+
+                                if (!xs_is_null(actor))
+                                    follower_add(&snac, actor);
+                            }
+                        }
+                    }
+
+                    user_free(&snac);
+                }
+            }
+
+            nf = 2.5;
+        }
 
         if (f < nf) {
             f          = nf;
diff --git a/utils.c b/utils.c
index 58c2246..84b367f 100644
--- a/utils.c
+++ b/utils.c
@@ -237,8 +237,9 @@ int adduser(char *uid)
     }
 
     const char *dirs[] = {
-        "actors", "followers", "following", "local", "muted",
-        "queue", "static", "timeline", "history", NULL };
+        "followers", "following", "local", "muted", "hidden",
+        "public", "private", "queue", "history",
+        "static", "timeline", NULL };
     int n;
 
     for (n = 0; dirs[n]; n++) {
diff --git a/xs.h b/xs.h
index da853f5..9cae3dc 100644
--- a/xs.h
+++ b/xs.h
@@ -34,6 +34,9 @@ typedef char d_char;
 /* auto-destroyable strings */
 #define xs __attribute__ ((__cleanup__ (_xs_destroy))) d_char
 
+/* not really all, just very much */
+#define XS_ALL 0xfffffff
+
 void *xs_free(void *ptr);
 void *_xs_realloc(void *ptr, size_t size, const char *file, int line, const char *func);
 #define xs_realloc(ptr, size) _xs_realloc(ptr, size, __FILE__, __LINE__, __FUNCTION__)
@@ -74,7 +77,7 @@ d_char *xs_list_pop(d_char *list, char **data);
 int xs_list_in(char *list, const char *val);
 d_char *xs_join(char *list, const char *sep);
 d_char *xs_split_n(const char *str, const char *sep, int times);
-#define xs_split(str, sep) xs_split_n(str, sep, 0xfffffff)
+#define xs_split(str, sep) xs_split_n(str, sep, XS_ALL)
 d_char *xs_dict_new(void);
 d_char *xs_dict_append_m(d_char *dict, const char *key, const char *mem, int dsz);
 #define xs_dict_append(dict, key, data) xs_dict_append_m(dict, key, data, xs_size(data))
diff --git a/xs_glob.h b/xs_glob.h
index c5293dc..978c200 100644
--- a/xs_glob.h
+++ b/xs_glob.h
@@ -5,7 +5,7 @@
 #define _XS_GLOB_H
 
 d_char *xs_glob_n(const char *spec, int basename, int reverse, int max);
-#define xs_glob(spec, basename, reverse) xs_glob_n(spec, basename, reverse, 0xfffffff)
+#define xs_glob(spec, basename, reverse) xs_glob_n(spec, basename, reverse, XS_ALL)
 
 
 #ifdef XS_IMPLEMENTATION
diff --git a/xs_io.h b/xs_io.h
index 9649484..6112dcb 100644
--- a/xs_io.h
+++ b/xs_io.h
@@ -79,7 +79,7 @@ d_char *xs_read(FILE *f, int *sz)
 d_char *xs_readall(FILE *f)
 /* reads the rest of the file into a string */
 {
-    int size = 0xfffffff;
+    int size = XS_ALL;
 
     return xs_read(f, &size);
 }
diff --git a/xs_regex.h b/xs_regex.h
index e0d1b7a..302bcf0 100644
--- a/xs_regex.h
+++ b/xs_regex.h
@@ -5,9 +5,9 @@
 #define _XS_REGEX_H
 
 d_char *xs_regex_split_n(const char *str, const char *rx, int count);
-#define xs_regex_split(str, rx) xs_regex_split_n(str, rx, 0xfffffff)
+#define xs_regex_split(str, rx) xs_regex_split_n(str, rx, XS_ALL)
 d_char *xs_regex_match_n(const char *str, const char *rx, int count);
-#define xs_regex_match(str, rx) xs_regex_match_n(str, rx, 0xfffffff)
+#define xs_regex_match(str, rx) xs_regex_match_n(str, rx, XS_ALL)
 
 #ifdef XS_IMPLEMENTATION
 
diff --git a/xs_set.h b/xs_set.h
index bd1b8ea..f97eb20 100644
--- a/xs_set.h
+++ b/xs_set.h
@@ -12,6 +12,7 @@ typedef struct _xs_set {
 } xs_set;
 
 void xs_set_init(xs_set *s);
+d_char *xs_set_result(xs_set *s);
 void xs_set_free(xs_set *s);
 int xs_set_add(xs_set *s, const char *data);
 
@@ -32,11 +33,21 @@ void xs_set_init(xs_set *s)
 }
 
 
-void xs_set_free(xs_set *s)
-/* frees a set */
+d_char *xs_set_result(xs_set *s)
+/* returns the set as a list and frees it */
 {
+    d_char *list = s->list;
+    s->list = NULL;
     s->hash = xs_free(s->hash);
-    s->list = xs_free(s->list);
+
+    return list;
+}
+
+
+void xs_set_free(xs_set *s)
+/* frees a set, dropping the list */
+{
+    free(xs_set_result(s));
 }
 
 
diff --git a/xs_version.h b/xs_version.h
index ac5d43f..baefcba 100644
--- a/xs_version.h
+++ b/xs_version.h
@@ -1 +1 @@
-/* a78beb97d364ff31cbaa504e275118afeaea7a59 */
+/* c18371e1f1d3de0f872354f93024a736caebea4d */