Activated hashtag RSS polling.

5 files changed, 43 insertions, 17 deletions

diff --git a/activitypub.c b/activitypub.c
index 0cc7bcb..2cffeac 100644
--- a/activitypub.c
+++ b/activitypub.c
@@ -3050,6 +3050,10 @@ void process_queue_item(xs_dict *q_item)
         }
     }
     else
+    if (strcmp(type, "rss_poll") == 0) {
+        rss_poll_hashtags();
+    }
+    else
         srv_log(xs_fmt("unexpected q_item type '%s'", type));
 }
 
diff --git a/html.c b/html.c
index 7fcc266..3d213a2 100644
--- a/html.c
+++ b/html.c
@@ -1601,7 +1601,8 @@ xs_html *html_top_controls(snac *user)
                     xs_html_attr("name", "followed_hashtags"),
                     xs_html_attr("cols", "40"),
                     xs_html_attr("rows", "4"),
-                    xs_html_attr("placeholder", "#cats\n#windowfriday\n#classicalmusic"),
+                    xs_html_attr("placeholder", "#cats\n#windowfriday\n#classicalmusic\nhttps:/"
+                        "/mastodon.social/tags/dogs"),
                     xs_html_text(followed_hashtags)),
 
                 xs_html_tag("br", NULL),
diff --git a/httpd.c b/httpd.c
index 41b2515..c94a542 100644
--- a/httpd.c
+++ b/httpd.c
@@ -705,34 +705,36 @@ static pthread_cond_t  sleep_cond;
 static void *background_thread(void *arg)
 /* background thread (queue management and other things) */
 {
-    time_t purge_time;
+    time_t t, purge_time, rss_time;
 
     (void)arg;
 
+    t = time(NULL);
+
     /* first purge time */
-    purge_time = time(NULL) + 10 * 60;
+    purge_time = t + 10 * 60;
+
+    /* first RSS polling time */
+    rss_time = t + 15 * 60;
 
     srv_log(xs_fmt("background thread started"));
 
     while (p_state->srv_running) {
-        time_t t;
         int cnt = 0;
 
         p_state->th_state[0] = THST_QUEUE;
 
         {
             xs *list = user_list();
-            char *p;
             const char *uid;
 
             /* process queues for all users */
-            p = list;
-            while (xs_list_iter(&p, &uid)) {
-                snac snac;
+            xs_list_foreach(list, uid) {
+                snac user;
 
-                if (user_open(&snac, uid)) {
-                    cnt += process_user_queue(&snac);
-                    user_free(&snac);
+                if (user_open(&user, uid)) {
+                    cnt += process_user_queue(&user);
+                    user_free(&user);
                 }
             }
         }
@@ -740,8 +742,10 @@ static void *background_thread(void *arg)
         /* global queue */
         cnt += process_queue();
 
+        t = time(NULL);
+
         /* time to purge? */
-        if ((t = time(NULL)) > purge_time) {
+        if (t > purge_time) {
             /* next purge time is tomorrow */
             purge_time = t + 24 * 60 * 60;
 
@@ -750,6 +754,22 @@ static void *background_thread(void *arg)
             job_post(q_item, 0);
         }
 
+        /* time to poll the RSS? */
+        if (t > rss_time) {
+            /* next RSS poll time */
+            int hours = xs_number_get(xs_dict_get_def(srv_config, "rss_poll_hours", "4"));
+
+            /* don't hammer servers too much */
+            if (hours < 1)
+                hours = 1;
+
+            rss_time = t + 60 * 60 * hours;
+
+            xs *q_item = xs_dict_new();
+            q_item = xs_dict_append(q_item, "type", "rss_poll");
+            job_post(q_item, 0);
+        }
+
         if (cnt == 0) {
             /* sleep 3 seconds */
 
diff --git a/rss.c b/rss.c
index b31ffd3..ce3e184 100644
--- a/rss.c
+++ b/rss.c
@@ -110,6 +110,9 @@ xs_str *rss_from_timeline(snac *user, const xs_list *timeline,
 void rss_to_timeline(snac *user, const char *url)
 /* reads an RSS and inserts all ActivityPub posts into the user's timeline */
 {
+    if (!xs_startswith(url, "https:/") && !xs_startswith(url, "http:/"))
+        return;
+
     xs *hdrs = xs_dict_new();
     hdrs = xs_dict_set(hdrs, "accept",     "application/rss+xml");
     hdrs = xs_dict_set(hdrs, "user-agent", USER_AGENT);
@@ -204,10 +207,9 @@ void rss_to_timeline(snac *user, const char *url)
 }
 
 
-void rss_process(void)
+void rss_poll_hashtags(void)
 /* parses all RSS from all users */
 {
-#if 0
     xs *list = user_list();
     const char *uid;
 
@@ -215,7 +217,7 @@ void rss_process(void)
         snac user;
 
         if (user_open(&user, uid)) {
-            const xs_list *rss = xs_dict_get(user.config, "rss");
+            const xs_list *rss = xs_dict_get(user.config, "followed_hashtags");
 
             if (xs_is_list(rss)) {
                 const char *url;
@@ -227,5 +229,4 @@ void rss_process(void)
             user_free(&user);
         }
     }
-#endif
 }
diff --git a/snac.h b/snac.h
index db51a1c..06a36f1 100644
--- a/snac.h
+++ b/snac.h
@@ -464,4 +464,4 @@ const char *lang_str(const char *str, const snac *user);
 xs_str *rss_from_timeline(snac *user, const xs_list *timeline,
                         const char *title, const char *link, const char *desc);
 void rss_to_timeline(snac *user, const char *url);
-void rss_process(void);
+void rss_poll_hashtags(void);