Big data storage change.

5 files changed, 131 insertions, 108 deletions

diff --git a/activitypub.c b/activitypub.c
index 7f63310..8e173c5 100644
--- a/activitypub.c
+++ b/activitypub.c
@@ -879,7 +879,7 @@ int process_message(snac *snac, char *msg, char *req)
         if (xs_type(object) == XSTYPE_DICT)
             object = xs_dict_get(object, "id");
 
-        timeline_admire(snac, object, actor, 1);
+        timeline_admire(snac, msg, object, actor, 1);
         snac_log(snac, xs_fmt("new 'Like' %s %s", actor, object));
         do_notify = 1;
     }
@@ -900,7 +900,7 @@ int process_message(snac *snac, char *msg, char *req)
                 xs *who_o = NULL;
 
                 if (valid_status(actor_request(snac, who, &who_o))) {
-                    timeline_admire(snac, object, actor, 0);
+                    timeline_admire(snac, msg, object, actor, 0);
                     snac_log(snac, xs_fmt("new 'Announce' %s %s", actor, object));
                     do_notify = 1;
                 }
@@ -1094,7 +1094,7 @@ int activitypub_get_handler(d_char *req, char *q_path,
     else
     if (strcmp(p_path, "outbox") == 0) {
         xs *id = xs_fmt("%s/outbox", snac.actor);
-        xs *elems = local_list(&snac, 20);
+        xs *elems = timeline_list(&snac, "public", 20);
         xs *list = xs_list_new();
         msg = msg_collection(&snac, id);
         char *p, *v;
diff --git a/data.c b/data.c
index 51ee2c7..e9f3d93 100644
--- a/data.c
+++ b/data.c
@@ -116,6 +116,7 @@ void user_free(snac *snac)
     xs_free(snac->config);
     xs_free(snac->key);
     xs_free(snac->actor);
+    xs_free(snac->md5);
 }
 
 
@@ -154,6 +155,7 @@ int user_open(snac *snac, char *uid)
 
                     if ((snac->key = xs_json_loads(key_data)) != NULL) {
                         snac->actor = xs_fmt("%s/%s", srv_baseurl, uid);
+                        snac->md5   = xs_md5_hex(snac->actor, strlen(snac->actor));
                         ret = 1;
                     }
                     else
@@ -566,14 +568,31 @@ int object_del_if_unref(const char *id)
 }
 
 
+d_char *_object_metadata(const char *id, const char *idxsfx)
+/* returns the content of a metadata index */
+{
+    xs *fn = _object_fn(id);
+    fn = xs_replace_i(fn, ".json", idxsfx);
+    return index_list(fn, XS_ALL);
+}
+
+
 d_char *object_children(const char *id)
 /* returns the list of an object's children */
 {
-    xs *fn = _object_fn(id);
+    return _object_metadata(id, "_c.idx");
+}
 
-    fn = xs_replace_i(fn, ".json", "_c.idx");
 
-    return index_list(fn, XS_ALL);
+d_char *object_likes(const char *id)
+{
+    return _object_metadata(id, "_l.idx");
+}
+
+
+d_char *object_announces(const char *id)
+{
+    return _object_metadata(id, "_a.idx");
 }
 
 
@@ -818,35 +837,6 @@ d_char *timeline_get(snac *snac, char *fn)
 }
 
 
-d_char *_timeline_list(snac *snac, char *directory, int max)
-/* returns a list of the timeline filenames */
-{
-    xs *spec = xs_fmt("%s/%s/" "*.json", snac->basedir, directory);
-    int c_max;
-
-    /* maximum number of items in the timeline */
-    c_max = xs_number_get(xs_dict_get(srv_config, "max_timeline_entries"));
-
-    /* never more timeline entries than the configured maximum */
-    if (max > c_max)
-        max = c_max;
-
-    return xs_glob_n(spec, 0, 1, max);
-}
-
-
-d_char *timeline_list(snac *snac, int max)
-{
-    return _timeline_list(snac, "timeline", max);
-}
-
-
-d_char *local_list(snac *snac, int max)
-{
-    return _timeline_list(snac, "local", max);
-}
-
-
 d_char *_timeline_new_fn(snac *snac, char *id)
 /* creates a new filename */
 {
@@ -1050,7 +1040,7 @@ int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer
 }
 
 
-d_char *timeline_top_level(snac *snac, d_char *list)
+d_char *timeline_top_level(d_char *list)
 /* returns the top level md5 entries from this index */
 {
     d_char *tl = xs_list_new();
@@ -1094,7 +1084,26 @@ d_char *timeline_top_level(snac *snac, d_char *list)
 }
 
 
-void timeline_admire(snac *snac, char *id, char *admirer, int like)
+d_char *timeline_list(snac *snac, const char *idx_name, int max)
+/* returns a timeline */
+{
+    int c_max;
+
+    /* maximum number of items in the timeline */
+    c_max = xs_number_get(xs_dict_get(srv_config, "max_timeline_entries"));
+
+    /* never more timeline entries than the configured maximum */
+    if (max > c_max)
+        max = c_max;
+
+    xs *idx  = xs_fmt("%s/%s.idx", snac->basedir, idx_name);
+    xs *list = index_list_desc(idx, max);
+
+    return timeline_top_level(list);
+}
+
+
+void timeline_admire(snac *snac, char *o_msg, char *id, char *admirer, int like)
 /* updates a timeline entry with a new admiration */
 {
     xs *ofn = _timeline_find_fn(snac, id);
diff --git a/html.c b/html.c
index 63b54c3..4c1ca79 100644
--- a/html.c
+++ b/html.c
@@ -386,7 +386,8 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num)
 {
     char *id    = xs_dict_get(msg, "id");
     char *actor = xs_dict_get(msg, "attributedTo");
-    char *meta  = xs_dict_get(msg, "_snac");
+    xs *likes   = object_likes(id);
+    xs *boosts  = object_announces(id);
 
     xs *s   = xs_str_new(NULL);
     xs *md5 = xs_md5_hex(id, strlen(id));
@@ -407,20 +408,14 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num)
         s = xs_str_cat(s, s1);
     }
 
-    {
-        char *l;
-
-        l = xs_dict_get(meta, "liked_by");
-        if (xs_list_in(l, snac->actor) == -1) {
-            /* not already liked; add button */
-            s = html_button(s, "like", L("Like"));
-        }
+    if (xs_list_in(likes, snac->md5) == -1) {
+        /* not already liked; add button */
+        s = html_button(s, "like", L("Like"));
+    }
 
-        l = xs_dict_get(meta, "announced_by");
-        if (strcmp(actor, snac->actor) == 0 || xs_list_in(l, snac->actor) == -1) {
-            /* not already boosted or us; add button */
-            s = html_button(s, "boost", L("Boost"));
-        }
+    if (strcmp(actor, snac->actor) == 0 || xs_list_in(boosts, snac->md5) == -1) {
+        /* not already boosted or us; add button */
+        s = html_button(s, "boost", L("Boost"));
     }
 
     if (strcmp(actor, snac->actor) != 0) {
@@ -477,23 +472,20 @@ d_char *html_entry_controls(snac *snac, d_char *os, char *msg, int num)
 }
 
 
-d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, int level, int *num)
+d_char *html_entry(snac *snac, d_char *os, char *msg, int local, int level, int *num)
 {
     char *id    = xs_dict_get(msg, "id");
     char *type  = xs_dict_get(msg, "type");
-    char *meta  = xs_dict_get(msg, "_snac");
     char *actor;
     int sensitive = 0;
     char *v;
+    xs *likes  = NULL;
+    xs *boosts = NULL;
 
     /* do not show non-public messages in the public timeline */
     if (local && !is_msg_public(snac, msg))
         return os;
 
-    /* return if already seen */
-    if (xs_set_add(seen, id) == 0)
-        return os;
-
     xs *s = xs_str_new(NULL);
 
     /* top wrap */
@@ -522,6 +514,14 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
 
         return xs_str_cat(os, s);
     }
+    else
+    if (strcmp(type, "Note") != 0) {
+        s = xs_str_cat(s, "<div class=\"snac-post\">\n");
+
+        xs *s1 = xs_fmt("<p>%s</p>\n", type);
+
+        return xs_str_cat(os, s);
+    }
 
     /* bring the main actor */
     if ((actor = xs_dict_get(msg, "attributedTo")) == NULL)
@@ -536,14 +536,14 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
 
     /* if this is our post, add the score */
     if (xs_startswith(id, snac->actor)) {
-        int likes  = xs_list_len(xs_dict_get(meta, "liked_by"));
-        int boosts = xs_list_len(xs_dict_get(meta, "announced_by"));
+        likes  = object_likes(id);
+        boosts = object_announces(id);
 
         /* alternate emojis: %d &#128077; %d &#128257; */
 
         xs *s1 = xs_fmt(
             "<div class=\"snac-score\">%d &#9733; %d &#8634;</div>\n",
-            likes, boosts);
+            xs_list_len(likes), xs_list_len(boosts));
 
         s = xs_str_cat(s, s1);
     }
@@ -553,28 +553,46 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
 
         s = xs_str_cat(s, "<div class=\"snac-post\">\n");
 
-        /* print the origin of the post, if any */
-        if (!xs_is_null(p = xs_dict_get(meta, "referrer"))) {
+        if (boosts == NULL)
+            boosts = object_announces(id);
+
+        if (xs_list_len(boosts)) {
+            /* if somebody boosted this, show as origin */
+            p = xs_list_get(boosts, 0);
             xs *actor_r = NULL;
 
-            if (valid_status(actor_get(snac, p, &actor_r))) {
+            if (xs_list_in(boosts, snac->md5) != -1) {
+                /* we boosted this */
+                xs *s1 = xs_fmt(
+                    "<div class=\"snac-origin\">"
+                    "<a href=\"%s\">%s</a> %s</a></div>",
+                    snac->actor, xs_dict_get(snac->config, "name"), L("boosted")
+                );
+
+                s = xs_str_cat(s, s1);
+            }
+            else
+            if (valid_status(object_get_by_md5(p, &actor_r, NULL))) {
                 char *name;
 
                 if ((name = xs_dict_get(actor_r, "name")) == NULL)
                     name = xs_dict_get(actor_r, "preferredUsername");
 
-                xs *s1 = xs_fmt(
-                    "<div class=\"snac-origin\">"
-                    "<a href=\"%s\">%s</a> %s</div>\n",
-                    xs_dict_get(actor_r, "id"),
-                    name,
-                    L("boosted")
-                );
+                if (!xs_is_null(name)) {
+                    xs *s1 = xs_fmt(
+                        "<div class=\"snac-origin\">"
+                        "<a href=\"%s\">%s</a> %s</div>\n",
+                        xs_dict_get(actor_r, "id"),
+                        name,
+                        L("boosted")
+                    );
 
-                s = xs_str_cat(s, s1);
+                    s = xs_str_cat(s, s1);
+                }
             }
         }
-        else
+
+#if 0
         if (!xs_is_null((p = xs_dict_get(meta, "parent"))) && *p) {
             /* this may happen if any of the autor or the parent aren't here */
             xs *s1 = xs_fmt(
@@ -586,18 +604,6 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
             s = xs_str_cat(s, s1);
         }
         else
-        if (!xs_is_null((p = xs_dict_get(meta, "announced_by"))) &&
-            xs_list_in(p, snac->actor) != -1) {
-            /* we boosted this */
-            xs *s1 = xs_fmt(
-                "<div class=\"snac-origin\">"
-                "<a href=\"%s\">%s</a> %s</a></div>",
-                snac->actor, xs_dict_get(snac->config, "name"), L("boosted")
-            );
-
-            s = xs_str_cat(s, s1);
-        }
-        else
         if (!xs_is_null((p = xs_dict_get(meta, "liked_by"))) &&
             xs_list_in(p, snac->actor) != -1) {
             /* we liked this */
@@ -609,6 +615,7 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
 
             s = xs_str_cat(s, s1);
         }
+#endif
     }
     else
         s = xs_str_cat(s, "<div class=\"snac-child\">\n");
@@ -717,12 +724,11 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
         s = html_entry_controls(snac, s, msg, *num);
 
     /** children **/
-
-    char *children = xs_dict_get(meta, "children");
-    int left       = xs_list_len(children);
+    xs *children = object_children(id);
+    int left     = xs_list_len(children);
 
     if (left) {
-        char *id;
+        char *p, *cmd5;
 
         if (level < 4)
             s = xs_str_cat(s, "<div class=\"snac-children\">\n");
@@ -732,16 +738,18 @@ d_char *html_entry(snac *snac, d_char *os, char *msg, xs_set *seen, int local, i
         if (left > 3)
             s = xs_str_cat(s, "<details><summary>...</summary>\n");
 
-        while (xs_list_iter(&children, &id)) {
-            xs *chd = timeline_find(snac, id);
+        p = children;
+        while (xs_list_iter(&p, &cmd5)) {
+            xs *chd = NULL;
+            object_get_by_md5(cmd5, &chd, NULL);
 
             if (left == 3)
                 s = xs_str_cat(s, "</details>\n");
 
             if (chd != NULL)
-                s = html_entry(snac, s, chd, seen, local, level + 1, num);
+                s = html_entry(snac, s, chd, local, level + 1, num);
             else
-                snac_debug(snac, 2, xs_fmt("cannot read from timeline child %s", id));
+                snac_debug(snac, 2, xs_fmt("cannot read from timeline child %s", cmd5));
 
             left--;
         }
@@ -773,13 +781,10 @@ d_char *html_timeline(snac *snac, char *list, int local)
 /* returns the HTML for the timeline */
 {
     d_char *s = xs_str_new(NULL);
-    xs_set seen;
     char *v;
     double t = ftime();
     int num = 0;
 
-    xs_set_init(&seen);
-
     s = html_user_header(snac, s, local);
 
     if (!local)
@@ -789,9 +794,12 @@ d_char *html_timeline(snac *snac, char *list, int local)
     s = xs_str_cat(s, "<div class=\"snac-posts\">\n");
 
     while (xs_list_iter(&list, &v)) {
-        xs *msg = timeline_get(snac, v);
+        xs *msg = NULL;
+
+        if (!valid_status(object_get_by_md5(v, &msg, NULL)))
+            continue;
 
-        s = html_entry(snac, s, msg, &seen, local, 0, &num);
+        s = html_entry(snac, s, msg, local, 0, &num);
     }
 
     s = xs_str_cat(s, "</div>\n");
@@ -830,8 +838,6 @@ d_char *html_timeline(snac *snac, char *list, int local)
 
     s = xs_str_cat(s, "</body>\n</html>\n");
 
-    xs_set_free(&seen);
-
     return s;
 }
 
@@ -1007,7 +1013,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char *
             status  = 200;
         }
         else {
-            xs *list = local_list(&snac, XS_ALL);
+            xs *list = timeline_list(&snac, "public", XS_ALL);
 
             *body   = html_timeline(&snac, list, 1);
             *b_size = strlen(*body);
@@ -1033,7 +1039,9 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char *
             else {
                 snac_debug(&snac, 1, xs_fmt("building timeline"));
 
-                xs *list = timeline_list(&snac, XS_ALL);
+                xs *list = timeline_list(&snac, "private", XS_ALL);
+
+                printf("--> %d\n", xs_list_len(list));
 
                 *body   = html_timeline(&snac, list, 0);
                 *b_size = strlen(*body);
@@ -1098,7 +1106,7 @@ int html_get_handler(d_char *req, char *q_path, char **body, int *b_size, char *
     if (strcmp(p_path, ".rss") == 0) {
         /* public timeline in RSS format */
         d_char *rss;
-        xs *elems = local_list(&snac, 20);
+        xs *elems = timeline_list(&snac, "public", 20);
         xs *bio   = not_really_markdown(xs_dict_get(snac.config, "bio"));
         char *p, *v;
 
@@ -1281,13 +1289,13 @@ int html_post_handler(d_char *req, char *q_path, d_char *payload, int p_size,
         if (strcmp(action, L("Like")) == 0) {
             xs *msg = msg_admiration(&snac, id, "Like");
             post(&snac, msg);
-            timeline_admire(&snac, id, snac.actor, 1);
+            timeline_admire(&snac, msg, id, snac.actor, 1);
         }
         else
         if (strcmp(action, L("Boost")) == 0) {
             xs *msg = msg_admiration(&snac, id, "Announce");
             post(&snac, msg);
-            timeline_admire(&snac, id, snac.actor, 0);
+            timeline_admire(&snac, msg, id, snac.actor, 0);
         }
         else
         if (strcmp(action, L("MUTE")) == 0) {
diff --git a/main.c b/main.c
index a109192..8fbdb88 100644
--- a/main.c
+++ b/main.c
@@ -167,7 +167,7 @@ int main(int argc, char *argv[])
 
         xs *idx  = xs_fmt("%s/private.idx", snac.basedir);
         xs *list = index_list_desc(idx, 256);
-        xs *tl   = timeline_top_level(&snac, list);
+        xs *tl   = timeline_top_level(list);
 
         xs *j    = xs_json_dumps_pp(tl, 4);
         printf("%s\n", j);
diff --git a/snac.h b/snac.h
index b8b5215..88f335a 100644
--- a/snac.h
+++ b/snac.h
@@ -33,6 +33,7 @@ typedef struct _snac {
     d_char *config;     /* user configuration */
     d_char *key;        /* keypair */
     d_char *actor;      /* actor url */
+    d_char *md5;        /* actor url md5 */
 } snac;
 
 int user_open(snac *snac, char *uid);
@@ -59,9 +60,14 @@ int index_first(const char *fn, char *buf, int size);
 d_char *index_list(const char *fn, int max);
 d_char *index_list_desc(const char *fn, int max);
 
+int object_get_by_md5(const char *md5, d_char **obj, const char *type);
 int object_del(const char *id);
 int object_del_if_unref(const char *id);
 
+d_char *object_children(const char *id);
+d_char *object_likes(const char *id);
+d_char *object_announces(const char *id);
+
 int follower_add(snac *snac, const char *actor);
 int follower_del(snac *snac, const char *actor);
 int follower_check(snac *snac, const char *actor);
@@ -73,11 +79,11 @@ d_char *_timeline_find_fn(snac *snac, char *id);
 d_char *timeline_find(snac *snac, char *id);
 int timeline_del(snac *snac, char *id);
 d_char *timeline_get(snac *snac, char *fn);
-d_char *timeline_list(snac *snac, int max);
+d_char *timeline_list(snac *snac, const char *idx_name, int max);
 int timeline_add(snac *snac, char *id, char *o_msg, char *parent, char *referrer);
-void timeline_admire(snac *snac, char *id, char *admirer, int like);
+void timeline_admire(snac *snac, char *o_msg, char *id, char *admirer, int like);
 
-d_char *timeline_top_level(snac *snac, d_char *list);
+d_char *timeline_top_level(d_char *list);
 
 d_char *local_list(snac *snac, int max);