web: fixed profile fetching to be seq

web/index.html

@@ -19,7 +19,6 @@
 		<script defer src="js/nostr.js?v=7"></script>
 		<script defer src="js/core.js?v=1"></script>
 		<script defer src="js/model.js?v=1"></script>
-		<script defer src="js/filters.js?v=1"></script>
 		<script defer src="js/contacts.js?v=1"></script>
 		<script defer src="js/event.js?v=1"></script>
 		<script defer src="js/unknowns.js?v=1"></script>

web/js/contacts.js

@@ -28,10 +28,6 @@ async function contacts_save(contacts) {
 		const db = ev.target.result;
 		let tx = db.transaction("friends", "readwrite");
 		let store = tx.objectStore("friends");
-		contacts.friends.forEach((pubkey) => {
-			//log_debug("storing", pubkey);
-			store.put({pubkey});
-		});
 		tx.oncomplete = (ev) => {
 			db.close();
 			resolve();
@@ -43,6 +39,13 @@ async function contacts_save(contacts) {
 			window.alert("An error occured saving contacts. Check console.");
 			reject(ev);
 		};
+
+		store.clear().onsuccess = () => {
+		 	contacts.friends.forEach((pubkey) => {
+		 		//log_debug("storing", pubkey);
+		 		store.put({pubkey});
+		 	});
+		};
 	}
 	return dbcall(_contacts_save);
 }

web/js/damus.js

@@ -10,7 +10,10 @@ const BOOTSTRAP_RELAYS = [
 const IMG_EVENT_LIKED = "icon/event-liked.svg";
 const IMG_EVENT_LIKE  = "icon/event-like.svg";
 
-const SID_PROFILES = "profiles";
+const SID_META = "meta";
+const SID_HISTORY = "history";
+const SID_NOTIFICATIONS = "notifications";
+const SID_EXPLORE = "explore";
 
 async function damus_web_init() {
 	init_message_textareas();
@@ -64,7 +67,7 @@ async function damus_web_init_ready() {
 	// Load all events from storage and re-process them so that apply correct
 	// effects.
 	await model_load_events(model, (ev)=> {
-		model_process_event(model, ev);
+		model_process_event(model, undefined, ev);
 	});
 	log_debug("loaded events", Object.keys(model.all_events).length);
 
@@ -107,7 +110,23 @@ function on_timer_save() {
 function on_pool_open(relay) {
 	log_info(`OPEN(${relay.url})`);
 	const model = DAMUS;
-	relay.subscribe(model.ids.account, filter_new_initial(model.pk));
+	const { pubkey } = model;
+
+	// Get all our info & history, well close this after we get  it
+	fetch_profile(pubkey, model.pool, relay);
+
+	// Get our notifications. We will never close this.
+	relay.subscribe(SID_NOTIFICATIONS, [{
+		kinds: STANDARD_KINDS,
+		"#p": [pubkey],
+		limit: 5000,
+	}]);
+
+	// Subscribe to the relay's world. We will also never close this.
+	relay.subscribe(SID_EXPLORE, [{
+		kinds: STANDARD_KINDS,
+		limit: 10000, // TODO this is a lot to handle and we should deal with it another way
+	}]);
 }
 
 function on_pool_notice(relay, notice) {
@@ -119,28 +138,15 @@ function on_pool_notice(relay, notice) {
 async function on_pool_eose(relay, sub_id) {
 	log_info(`EOSE(${relay.url}): ${sub_id}`);
 	const model = DAMUS;
-	const { ids, pool } = model;
+	const { pool } = model;
-
-	if (sub_id.indexOf(SID_PROFILES) == 0) {
-		model.pool.unsubscribe(sub_id, relay);
-		request_profiles(model);
-		return;
-	}
 	
+	const sid = sub_id.slice(0, sub_id.indexOf(":"));
 	switch (sub_id) {
-		case ids.home:
+		case SID_META:
-			pool.unsubscribe(ids.home, relay);
+			model_get_relay_que(model, relay).busy = false;
-			if (!model.inited) {
+			model_fetch_next_profile(model, relay);
-				model.inited = true;
+		case SID_HISTORY:
-			}
+			pool.unsubscribe(sub_id, relay);
-			break;
-		case ids.unknown:
-			pool.unsubscribe(ids.unknowns, relay);
-			break;
-		case ids.account:
-			model.done_init[relay] = true;
-			pool.unsubscribe(ids.account, relay);
-			model_subscribe_defaults(model, relay);
 			break;
 	}
 }
@@ -156,13 +162,29 @@ function on_pool_event(relay, sub_id, ev) {
 	if (new Date(ev.created_at * 1000) > new Date()) {
 		return;	
 	}
-	model_process_event(model, ev);
+	model_process_event(model, relay, ev);
+}
 
-	// Request the profile if we have never seen it
+function fetch_profile_info(pubkey, pool, relay) {
-	if (!model.profile_events[ev.pubkey]) {
+	pool.subscribe(`${SID_META}:${pubkey}`, [{
-		model.requested_profiles.push({relay, pubkey: ev.pubkey});
+		kinds: [KIND_METADATA, KIND_CONTACT],
-		request_profiles(model);
+		authors: [pubkey],
-	}
+		limit: 1,
+	}], relay);
+}
+
+function fetch_profile(pubkey, pool, relay) {
+	fetch_profile_info(pubkey, pool, relay);	
+	pool.subscribe(`${SID_HISTORY}:${pubkey}`, [{
+		kinds: STANDARD_KINDS,
+		authors: [pubkey],
+		limit: 1000,
+	}], relay);
+}
+
+/*
+function new_sub_id(prefix) {
+	return `${prefix}:${uuidv4()}`;
 }
 
 let request_profiles_timer;
@@ -205,9 +227,6 @@ function fetch_queued_profiles(model) {
 		log_debug(`(${relay.url}) fetching profiles ${sid} size(${set.size})`);
 	})
 	return model.requested_profiles.length > 0;
-}
+}*/
 
-function new_sub_id(prefix) {
-	return `${prefix}:${uuidv4()}`;
-}
 

web/js/filters.js

@@ -1,107 +0,0 @@
-function filters_subscribe(filters, pool, relays=undefined) {
-	for (const key in filters) {
-		pool.subscribe(key, filters[key], relays);
-	}
-}
-
-function filters_new_default(model) {
-	const { pubkey, ids, contacts } = model;
-	const friends = Array.from(contacts);	
-	friends.push(pubkey);
-	const f = {};
-	f[ids.home] = filters_new_friends(friends);
-	//f[ids.home] = [{authors: [pubkey], kinds: STANDARD_KINDS}];
-	f[ids.contacts] = filters_new_contacts(friends);
-	f[ids.dms] = filters_new_dms(pubkey);
-	f[ids.notifications] = filters_new_notifications(pubkey);
-	f[ids.explore] = [{kinds: STANDARD_KINDS, limit: 500}];
-	return f;
-}
-
-function filters_new_default_since(model, cache) {
-	const filters = filters_new_default(model);
-	for (const key in filters) {
-		filters[key] = filters_set_since(filters[key], cache);
-	}
-	return filters;
-}
-
-function filter_new_initial(pubkey) {
-	return {
-		authors: [pubkey],
-		kinds: [KIND_CONTACT],
-		limit: 1,
-	};
-}
-
-function filters_new_contacts(friends) {
-	return [{
-		kinds: [KIND_METADATA],
-		authors: friends,
-	}]
-}
-
-function filters_new_dms(pubkey, limit=100) {
-	return [
-	{ // dms we sent
-		kinds: [KIND_DM],
-		limit,
-		authors: [pubkey],
-	}, { // replys to us
-		kinds: [KIND_DM],
-		limit: limit,
-		"#p":  [pubkey],
-	}]
-}
-
-function filters_new_friends(friends, limit=500) {
-	return [{
-		kinds: STANDARD_KINDS,
-		authors: friends,
-		limit,
-	}]
-}
-
-function filters_new_notifications(pubkey, limit=100) {
-	return [{
-		kinds: STANDARD_KINDS,
-		"#p": [pubkey],
-		limit,
-	}]
-}
-
-function filters_set_since(filters=[], cache={}) {
-	filters.forEach((filter) => {
-		const since = get_earliest_since_time(filter.kinds, cache)
-		delete filter.since;
-		if (since) filter.since = since;
-	});
-	return filters
-}
-
-function get_earliest_since_time(kinds=[], cache={}) {
-	const earliest = kinds.reduce((a, kind) => {
-		const b = get_since_time(cache[kind]);
-		if (!a) {
-			return b;
-		}
-		return b < a ? b : a;
-	}, undefined);
-	return earliest;
-}
-
-/* calculate_last_of_kind returns a map of kinds to time, where time is the 
- * last time it saw that kind.
- */
-function calculate_last_of_kind(evs) {
-	const now_sec = new Date().getTime() / 1000
-	return Object.keys(evs).reduce((obj, evid) => {
-		const ev = evs[evid]
-		if (!is_valid_time(now_sec, ev.created_at))
-			return obj
-		const prev = obj[ev.kind] || 0
-		obj[ev.kind] = get_since_time(max(ev.created_at, prev))
-		return obj
-	}, {})
-}
-

web/js/model.js

@@ -2,7 +2,7 @@
  * a relay. Additionally other side effects happen such as notification checks
  * and fetching of unknown pubkey profiles.
  */
-function model_process_event(model, ev) {
+function model_process_event(model, relay, ev) {
 	if (model.all_events[ev.id]) {
 		return;
 	}
@@ -16,10 +16,10 @@ function model_process_event(model, ev) {
 	let fn;
 	switch(ev.kind) {
 		case KIND_METADATA:
-			fn = model_process_event_profile;
+			fn = model_process_event_metadata;
 			break;
 		case KIND_CONTACT:
-			fn = model_process_event_contact;
+			fn = model_process_event_following;
 			break;
 		case KIND_DELETE:
 			fn = model_process_event_deletion;
@@ -29,47 +29,62 @@ function model_process_event(model, ev) {
 			break;
 	}
 	if (fn)
-		fn(model, ev);
+		fn(model, ev, !!relay);
-
-	// check if the event that just came in should notify the user and is newer
-	// than the last recorded notification event, if it is notify
-	const notify_user = event_refs_pubkey(ev, model.pubkey)
-	const last_notified = get_local_state('last_notified_date')
-	ev.notified = notify_user;
-	if (notify_user && (last_notified == null || ((ev.created_at*1000) > last_notified))) {
-		set_local_state('last_notified_date', new Date().getTime());
-		model.notifications++;
-		update_title(model);
-	}
-
-	// If we find some unknown ids lets schedule their subscription for info
-	if (model_event_has_unknown_ids(model, ev))
-		schedule_unknown_refetch(model);
 
 	// Queue event for rendering  
 	model.invalidated.push(ev.id);
+
+	// If the processing did not come from a relay, but instead storage then
+	// let us simply ignore fetching new things.
+	if (!relay)
+		return;
+
+	// Request the profile if we have never seen it
+	if (!model.profile_events[ev.pubkey])
+		model_fetch_next_profile(model, relay, ev.pubkey);
+	
+	// TODO fetch unknown referenced events & pubkeys from this event
+	// TODO notify user of new events aimed at them!
+}
+
+function model_get_relay_que(model, relay) {
+	return map_get(model.relay_que, relay, {profiles:[]});
+}
+
+function model_fetch_next_profile(model, relay, pubkey) {
+	const que = model_get_relay_que(model, relay);
+	if (pubkey)
+		que.profiles.push(pubkey);
+	if (que.busy || que.profiles.length == 0)
+		return;
+	que.busy = true;
+	fetch_profile_info(que.profiles.shift(), model.pool, relay);
 }
 
 /* model_process_event_profile updates the matching profile with the contents found 
  * in the event.
  */
-function model_process_event_profile(model, ev) {
+function model_process_event_metadata(model, ev, update_view) {
 	const prev_ev = model.all_events[model.profile_events[ev.pubkey]]
 	if (prev_ev && prev_ev.created_at > ev.created_at)
 		return
 	model.profile_events[ev.pubkey] = ev.id
 	model.profiles[ev.pubkey] = safe_parse_json(ev.content, "profile contents")
+	if (update_view)	
 		view_timeline_update_profiles(model, ev); 
 }
 
-function model_process_event_contact(model, ev) {
+function model_process_event_following(model, ev, update_view) {
 	contacts_process_event(model.contacts, model.pubkey, ev)
-	load_our_relays(model.pubkey, model.pool, ev)
+	// TODO support loading relays that are stored on the initial relay
+	// I find this wierd since I may never want to do that and only have that
+	// information provided by the client - to be better understood
+//	load_our_relays(model.pubkey, model.pool, ev)
 }
 
 /* model_process_event_reaction updates the reactions dictionary
  */
-function model_process_event_reaction(model, ev) {
+function model_process_event_reaction(model, ev, update_view) {
 	let reaction = event_parse_reaction(ev);
 	if (!reaction) {
 		return;
@@ -77,18 +92,19 @@ function model_process_event_reaction(model, ev) {
 	if (!model.reactions_to[reaction.e])
 		model.reactions_to[reaction.e] = new Set();
 	model.reactions_to[reaction.e].add(ev.id);	
+	if (update_view)
 		view_timeline_update_reaction(model, ev);
 }
 
 /* event_process_deletion updates the list of deleted events. Additionally
  * pushes event ids onto the invalidated stack for any found.
  */
-function model_process_event_deletion(model, ev) {
+function model_process_event_deletion(model, ev, update_view) {
 	for (const tag of ev.tags) {
 		if (tag.length >= 2 && tag[0] === "e" && tag[1]) {
 			let evid = tag[1];
 			model.invalidated.push(evid);
-			model_remove_reaction(model, evid);
+			model_remove_reaction(model, evid, update_view);
 			if (model.deleted[evid])
 				continue;
 			let ds = model.deletions[evid] =
@@ -98,7 +114,7 @@ function model_process_event_deletion(model, ev) {
 	}
 }
 
-function model_remove_reaction(model, evid) {
+function model_remove_reaction(model, evid, update_view) {
 	// deleted_ev -> target_ev -> original_ev
 	// Here we want to find the original react event to and remove it from our
 	// reactions map, then we want to update the element on the page. If the 
@@ -112,6 +128,7 @@ function model_remove_reaction(model, evid) {
 		return;
 	if (model.reactions_to[reaction.e])
 		model.reactions_to[reaction.e].delete(target_ev.id);
+	if (update_view)
 		view_timeline_update_reaction(model, target_ev);
 }
 
@@ -182,25 +199,6 @@ function model_has_event(model, evid) {
 	return evid in model.all_events
 }
 
-/* model_relay_update_lok returns a map of kinds found in all events based on 
- * the last seen event of each kind. It also updates the model's cached value.
- * If the cached value is found it returns that instead
- */
-function model_relay_update_lok(model, relay) {
-	let last_of_kind = model.last_event_of_kind[relay];
-	if (!last_of_kind) {
-		last_of_kind = model.last_event_of_kind[relay] 
-			= calculate_last_of_kind(model.all_events);
-	}
-	return last_of_kind;
-}
-
-function model_subscribe_defaults(model, relay) {
-	const lok = model_relay_update_lok(model, relay);
-	const filters = filters_new_default_since(model, lok);
-	filters_subscribe(filters, model.pool, [relay]);
-}
-
 function model_events_arr(model) {
 	const events = model.all_events;
 	let arr = [];
@@ -309,18 +307,6 @@ function new_model() {
 		},
 		invalidated: [], // event ids which should be added/removed
 		elements: {}, // map of evid > rendered element
-		requested_profiles: [], // an array of {relay_id, pubkey} to fetching
+		relay_que: new Map(),
-
-		ids: {
-			comments:      "comments",
-			explore:       "explore",
-			refevents:     "refevents",
-			account:       "account",
-			home:          "home",
-			contacts:      "contacts",
-			notifications: "notifications",
-			unknowns:      "unknowns",
-			dms:           "dms",
-		},
 	};
 }

web/js/ui/state.js

@@ -66,6 +66,10 @@ function view_timeline_apply_mode(model, mode, opts={}) {
 		view_show_spinner(false);
 	}
 
+	// Request the background info for this user
+	if (pubkey)
+		fetch_profile(pubkey, model.pool);
+
 	return mode;
 }
 

web/js/util.js

@@ -74,6 +74,13 @@ function arr_bsearch_insert(arr, item, cmp) {
 	return start;
 }
 
+function map_get(m, k, def) {
+	if (!m.has(k)) {
+		m.set(k, def);
+	}
+	return m.get(k);
+}
+
 function is_valid_time(now_sec, created_at) {
 	// don't count events far in the future
 	if (created_at - now_sec >= 120) {