Mnemosyne data analysis

I did this a while ago.

Steps:

1. Download 2014-01-27-mnemosynelogs-all.db.xz
2. Extract: tar -xvf 2014-01-27-mnemosynelogs-all.db.xz
3. Count the records: sqlite3 -batch "select event, count(*) From log Group by event;"
   
   

   Total	121 188 408	Event type
   1	813 548	start
   2	786 586	stop
   3	172 462	scheduler
   4	3 123 878	load db
   5	1 109 352	save db
   6	58 532 022	add card
   8	7 684 724	delete card
   9	48 965 836	repetition

4. Dump repetitions to CSV: sqlite3 -csv -header -batch 2014-01-27-mnemosynelogs-all.db "select object_id,grade,acq_reps,ret_reps,actual_interval From log where event=9 limit 7;" (this only does the first few lines of course)
   
   

   c136315a,9779a1ad,1,0,0,5
   c136315a,a2a80b21,1,0,0,4
   c136315a,a35a6c4a,1,0,0,5
   c136315a,85f8ec88,1,0,0,5
   c136315a,10ae2adc,1,0,0,5
   c136315a,a9c66681,1,0,0,4
   c136315a,ba841422,1,0,0,5
   c136315a,4e108d46,1,0,0,5
   

5. Sort on time to completion and grade:
   

   db9864c5,37132b6b,6,10,-188957807,3
   db9864c5,680c6178,3,7,-188879971,3
   db9864c5,40712c82,7,6,-188879963,3
   db9864c5,a5679e62,17,14,-188879941,3
   db9864c5,21a9326f,14,11,-188652473,3
   dQjTu8hqz3d04ReDrWTxdZ,7humHviBhzGDsbmfZzkFNh,2,0,-157680000,3
   dQjTu8hqz3d04ReDrWTxdZ,LGuN7QIDz7jsjuwKeSazrB,2,0,-157680000,3
   dQjTu8hqz3d04ReDrWTxdZ,a2cgcETy4U1mpR8gId581B,2,0,-157680000,2
   dQjTu8hqz3d04ReDrWTxdZ,2dHnVSa6v7HJXd1Bk6oOI7,2,0,-157680000,0
   dQjTu8hqz3d04ReDrWTxdZ,evdCglXJdRSwNAJwOaxDJo,2,0,-157680000,0
   dQjTu8hqz3d04ReDrWTxdZ,iWboGsLywTzw2agIQPUsBw,3,1,-157680000,0
   dQjTu8hqz3d04ReDrWTxdZ,y4hyxaB2HdW6UVQn15vdoh,23,2,-157680000,0
   qFHzllJkeSgoONQZJxkJ3c,LWYCS7wgH2T3aHPRUVUnH4,3,0,-127916114,2
   qFHzllJkeSgoONQZJxkJ3c,CIbwahRHaTaTCujq1X10QB,3,0,-127916114,0
   qFHzllJkeSgoONQZJxkJ3c,QPuSfCkJzbzk7uw2AJEA73,5,0,-127916103,1
   qFHzllJkeSgoONQZJxkJ3c,zdpemIGjakRrDMmK7Lvc6d,2,0,-127916095,1
   qFHzllJkeSgoONQZJxkJ3c,ILfYUuLjwER4VnpIYtUpWj,6,0,-127916088,0
   qFHzllJkeSgoONQZJxkJ3c,GdkgWUvUZMkhbp4QZim6fA,13,0,-127916071,1
   

6. Run this Haskell:
   

   {-# LANGUAGE ScopedTypeVariables #-}
   
   module GradeMunge where
   
   import qualified Data.ByteString.Lazy as BL
   import Data.Csv.Streaming
   import Data.Csv(encode)
   import System.IO
   import qualified Data.Map.Strict as M
   
   type Data = (Integer,M.Map Integer Integer,(Integer,Integer))
   
   def :: Integer -> Data
   def r = (0,M.fromList [(0,0),(1,0),(2,0),(3,0),(4,0),(5,0)],(r,r))
   
   main :: IO ()
   main = do
       csvData <- BL.readFile "grades.csv"
       let out = munge (decode NoHeader csvData) (def (-188957808))
       BL.writeFile "gradehisto.csv" (encode out)
   
   munge :: Records (BL.ByteString, BL.ByteString, Integer, Integer, Integer, Integer) -> Data -> [[Integer]]
   munge (Cons (Right (user,obj,acq_reps,ret_reps,interval,grade)) k) d@(total,_,(low,high)) =
       if (total < 10000 || (low <= interval && interval <= high && total < 100000)) then
           -- add the record
           munge k (insertRecord interval grade d)
       else
           -- return the record and start a new one
           prepOut d : munge k (insertRecord interval grade (def interval))
   munge (Cons (Left err) k) a = error ("blah: " ++ err) -- munge k a
   munge (Nil (Just s) k') a = error ("nil: " ++ s) -- [a]
   munge (Nil Nothing k') a = [prepOut a]
   
   prepOut :: Data -> [Integer]
   prepOut d@(total,xs,(low,high)) | length (M.keys xs) == 6 = low : high : total : M.elems xs
                                 | otherwise = error (show d ++ " keys: " ++ show (M.keys xs))
   
   insertRecord :: Integer -> Integer -> Data -> Data
   insertRecord interval grade (total,xs,(low,high)) = (total+1,M.insertWith (+) grade 1 xs,(low `min` interval,high `max` interval))
   

7. Plot in Mathematica:
   
   

And my notes on the database format:

filename: (user_id)_[(machine_id)_](log_number).txt
CREATE TABLE parsed_logs(log_name text primary key); # used for incremental processing
CREATE TABLE _cards(id text primary key,last_rep int,offset int); # used for version < 2 munging; 'last_rep' is the timestamp of the card's last grading time; offset is 0, 1 (grade>=2 phase 1), or -1 (grade <= 2 phase 2)
insert or replace into _cards(id=card_id + user_id, offset, last_rep=)

CREATE TABLE log(
        user_id text,
        event integer,
        timestamp integer,
        object_id text,
        grade integer,
        easiness real,
        acq_reps integer,
        ret_reps integer,
        lapses integer,
        acq_reps_since_lapse integer,
        ret_reps_since_lapse integer,
        scheduled_interval integer,
        actual_interval integer,
        thinking_time integer,
        next_rep integer
    );
# Program Started (program_name_version = Mnemosyne 1.0-RC nt win32)
insert into log(user_id, event=STARTED_PROGRAM=1, timestamp, object_id=program_name_version)
# Program stopped
insert into log(user_id, event=STOPPED_PROGRAM=2, timestamp)
# Scheduler SM2 Mnemosyne
insert into log(user_id, event=STARTED_SCHEDULER=3, timestamp, object_id=scheduler_name)
# Loaded database N N N
insert into log(user_id, event=LOADED_DATABASE=4, timestamp, object_id=machine_id, acq_reps=scheduled_count, ret_reps=non_memorised_count, lapses=active_count)
# Saved database N N N
insert into log(user_id, event=SAVED_DATABASE=5, timestamp, object_id=machine_id, acq_reps=scheduled_count, ret_reps=non_memorised_count, lapses=active_count)
# New item id grade new_interval (munged, possibly add repetition too)
# Imported item id grade ret_reps last_rep next_rep interval (not munged)
insert into log(user_id, event=ADDED_CARD=6, timestamp, object_id=card_id)
# Deleted item id
insert into log(user_id, event=DELETED_CARD=8, timestamp, object_id=card_id)
# R id grade easiness | acq_reps ret_reps lapses acq_reps_since_lapse ret_reps_since_lapse | scheduled_interval actual_interval | new_interval noise | thinking_time
# R id grade 2.5 | 1 0 0 1 0 | 0 0 | new_interval 0 | 0 when adding new card and grade >= 2
insert into log(user_id, event=REPETITION=9, timestamp, object_id=card_id, grade, easiness, acq_reps, ret_reps, lapses, acq_reps_since_lapse, ret_reps_since_lapse, scheduled_interval, actual_interval=timestamp-previous_rep_timestamp, thinking_time, next_rep=timestamp + new_interval)

initial grading in 'Add Card' counted as an acquisition repetition (and explicitly logged as an 'R' event) when grade 2+, otherwise grade= -1 like imported cards

card_id is created as a hash of the card data
gr= the grade, 0-5, default is 0, -1 means "unseen"
easyiness is the easiness parameter from the SM2 algorithm
acquisition reps = # w/ gr<2, including card
retention reps = # w/ gr=2+, including card
lapses = the number of times you forget this card (new grade < 2, old grade >=2)
ac_rp_l, rt_rp_l = ... since lapse
timestamp = last_rep = last actual repetition
next_rep = next scheduled repetition, timestamp
sch_i: scheduled previous interval in seconds
act_i: actual previous interval in seconds
th_t: thinking time in seconds
initial grade 0,1 (failing)   initial grade 2,3,4,5 (passing)
Version < 0.9.8 (phase 1) acq_reps=0    acq_reps=0, R added, acq++
0.9.8 <= version < 2.0 (phase 2) acq_reps=1, but all acq–    acq_reps=1->0, R added
2.0 <= version acq_reps=0    acq_reps=0, R

Workflow diagram

So apparently the web has moved on to HTML5; let's get some diagrams going.

Decisions

On unfamiliar ground, such as a changing industry or a new product or business launch, models can be dangerous. In the messy world we live in, they often make invalid assumptions or identify non-existent patterns. Few research efforts have focused on analyzing historical data or identifying similarities between phenomena (which can be considered replications of statistical tools). Thus, most statistical methods are untested and, at best, partial solutions, as the relevant variables and their relationships with outcomes often have no consensus. Although complex regression models have been developed, and shown to be somewhat accurate in their domain, they generally only consider a small subset of the factors that predict success. Even once a statistical method is proven and applicable, many challenges remain unsolved in visualization and user interaction. Interactively and quickly exploring algorithms, parameters, metadata, and data sets requires a large amount of functionality, including zooming, highlighting, filtering, clipping, sorting, smoothing, searching, plotting, focusing, lensing, and side-by-side visualizations. Few graphical toolkit libraries implement all of these or in a fashion that scales to large data sets.

Given this, many decisions still require humans for their consideration and conclusions. Unfortunately, humans are subject to many biases. When presented with a data sample, people naturally begin performing a comparison of the problem with the sample. Due to anchoring, even if the sample is irrelevant, it will still influence their decision. Due to effects such as backfires, bandwagons, decoys, and focusing, it can even influence their answer in the wrong direction. Supposing the data does have a deep correspondence with the problem at hand and creates the correct conclusion, there is still hindsight bias in computing the uncertainty of results and hyperbolic discounting of the payoffs of results.

Using multiple data samples encourages a statistical and historical view of the problem and thus produces more accurate forecasts. It compresses the natural try-fail cycle people use to formulate solutions, allowing people to learn from the past and determine relationships between determinants and outcomes. On average, using more data samples generates more strategies and ensures that the chosen strategy is more successful.

The similarity between samples and the problem has an interesting effect. When generating strategies, it is most useful to have a set of distantly-related or even unrelated samples to consider, as these generate better ideas. However, closely-related examples are a better basis for predicting performance or evaluating strategies, and can greatly reduce over-optimistic evaluations.

It is thus necessary to have both distantly- and closely-related samples for a creative and correct decision. Unfortunately, even when encouraged to do so, people are not naturally inclined to form a broad set of samples; they suffer recall or availability bias. Using a sample set selected randomly from a representative reference class by a unbiased method such as partitioning into intervals or deciles generally reduces this bias. Focusing on the relative similarity of samples and the aspects of them that are generalizable to multiple cases shifts the discussion towards the empirical facts and deep structures of the problem at hand, avoiding superficial details and their unwanted effects. Robust analogizing with differential attention to the most similar and least similar cases increases the precision of outcome analysis and greatly facilitates learning from other people's experiences.

The following procedure incorporates these ideas:

1. Define the problem and your purpose
1. predictors, relevant categories or potentialities
2. the plan/timeline/budget/milestone variables desired
3. What kind of comparison is being made
2. Generate a reference class of similar/analogous problems/cases, using as many analogies and references as possible
1. The class should be universal ("all possible states/choices/outcomes/things of the world")
2. Common analogies include position, diversity, resources, cycles, economics, and ecology
3. Make design choices about the study - data sources etc.
3. Select a set of samples from the reference class. The set should be unbiased and distribution-based, e.g. a "random subset".
1. predetermine the method of selection, using a rigorously structured approach
2. avoid memory sampling
3. avoid using a large set of samples - subsampling allows you to either measure the fragility of a conclusion (if the procedure is repeated) or reduce labor (if not)
4. Assess the source cases - research
1. strategies that were pursued, relevant qualities/quantities, and how they turned out ("results")
2. interaction effects between states, choices, and outcomes - historical analogies
5. Assess the similarity of the source cases to the target
1. subjective weighting
1. rank & rate for similarity - crowdsourcing survey, multiple observers, estimate observer reliability
2. aggregate with robust mean function - give small weight to outliers
3. can use non-experts or experts depending on domain / availability of info
2. rule-based similarity - if relevant features/importance are known
6. Construct an estimate of results, using the similarity weightings
1. create measure for comparison of outcome, e.g. money
2. obtain priors over the probability of the outcome occurring
3. hierarchical cluster analysis - avg. of ~6 - identify "top" cluster containing samples most similar to problem
4. estimate average outcomes of samples - use to predict problem
7. Assess the predictability of outcomes - regression on samples using estimate from (6) and other known variables
1. Make statistical corrections of the estimate
2. Adapt or translate results

Reply to Scott Walker

I was looking at Catholic Church and AIDS when I wrote "[the Church's] statements look naive and uninformed". Claiming that "The spermatozoon can easily pass through the 'net' that is formed by the condom.", when the CDC says "Laboratory studies have demonstrated that latex condoms provide an essentially impermeable barrier to particles the size of STD pathogens", seems like a clear example of ideology taking precedence over facts.

Also, I think the Church has a weak internet presence; for example, the Vatican's website is only the 4th result for "Catholic Church."

The Church has indeed had many scientists; but they are in the minority today. Pope Benedict XVI apparently desires to change this, but the fact that Catholic scientists are the minority in the first place means something in the way the Church works has gone askew. I'm not certain if, as you seem to suggest, being Catholic makes one perform better in math or science; but I think it's irrelevant; the problem is demographics, not talent.

As far as history, I was trying to show that man created, if not God himself, then all of the notions and words surrounding God. I'm fairly confident the trilobites didn't have a bible, or a language, or even coherent thoughts that could conceive of "God"; what has changed since then, other than the works of man? I have no direct experience of God; the evidence I have read shows no direct experience of God; therefore I conclude it is not possible to directly experience God; thus God is not relevant to my decisions; thus I have no beliefs about God and consider any statements about him to not be statements of fact but instead statements about something else. (This argument applies mostly to God; something physical, such as the Catholic Church, is directly experienceable, and thus factors into my decisions; it can go so far as to causing me to say that "I believe in God", where the spoken word "belief" is redefined from my preferred meaning of expectations about one's direct experience to some sort of signal of approval of the Catholic Church)

My standard of evidence is not the requirement of "photos, data-readings, and peer reviewed articles"; but the Bible has factual inaccuracies, so I don't think one can easily deduce that Adam existed from the Bible. I don't see any evidence for Adam other than the Bible; thus I conclude, that the Biblical Adam is not directly experienceable  and thus irrelevant. Similarly for dead people (not experienceable, although their books etc. might be), fictitious ideas, nonsense ideas, etc.

As for the Ten Commandments, they may be clear in some situations, e.g. "Thou shalt not kill", but I think most decisions are not clear-cut, and multiple rules apply. I don't see any procedures for weighing values against each other to decide which is more important; thus, I say that it is "non-specific". As for arguing with hypotheticals, the whole point is that it is an extreme situation. Let's flesh it out a bit; say you wake up from being unconscious on a bridge with no railings, after having been kidnapped by Islamic terrorists, and find a man standing in front of you (near the edge), holding an AK-47 and apparently about to shoot 5 hostages. Surely it's clear that you should push him off the bridge or at least attempt to disarm him? Now consider a similar case, indistinguishable from the previous unless you know the context; the man holding the AK-47 (which is unloaded) is actually a hostage, who has been told that a sniper will shoot him if he doesn't appear to be about to kill the other 5 people, who are actually more terrorists that will torture you if you don't push the man with the AK-47 off the bridge, but will otherwise send you back free to America as you have shown that you agree with their morality. This, clearly, is a much less likely scenario; but after just waking up, you have no way of knowing which is which; perhaps the situation is even more complicated than I described. But you must act instantly, or else the sniper will shoot you once he notices you are awake (you can see the sniper from where you are lying on the ground). Or consider a war; perhaps Private Jenkins (who has a suicide explosive pack) can be sacrificed to save Privates Adam, Blaire, Cain, Dominic, and Eric; but if he is not sacrificed, then everyone else will die a horrible death but Jenkins will be spared at least temporarily while they remove his explosive pack.

Even with all details specified, I don't see much from the Church on how to decide on a course of action. But e.g. utilitarianism can make those quick decisions: 5 people alive is better than 1, if nothing to distinguish those people is known. Perhaps utilitarianism is not a "humane morality"; but if the Church cannot even provide an answer, then I don't see how it can have any claim to have a theory of morality; if the Church has no theory of morality, then it is just another institution set loose in the world without guidance, and I don't see any reason to accord it any special status beyond that of, say, a multinational political party.

Ideas on UI

One of the columns missing from the last post was UI. That's mainly because I don't know enough about its history to divide it up into generations. But I think there are two general trends, namely in expressiveness (blinking lights → black-and-white → color → really big color, holographic, etc.), and in ubiquity (giant fixed → giant unfixed → portable → wearable). It's hard to say which of these will win out in the end, or indeed if they're in competition or rather that one is a lagged version of the other; but there is tension between a “Matrix-like” mind-machine interface and a “networked world” tool to access data. The question is whether the display takes an active or a passive role; does it modify the user's perceptions, or does it occupy the perceptual space?

This tension is probably most obvious in audio; the space is well-understood. A human has two audio inputs, a left ear and a right ear. Given a sufficiently expensive sound system, we can set those inputs to anything we like. We are given an audio environment, say at a music concert. What do we play? One trend says to play from the source: give the music as the music creator hears it, immersed in the mood. If the creator hears the flutes from the left, the synthesizers on the right, so be it, even if no such instruments are present when the recording is listened to. The other says to consider the context: a jogger is listening to the music merely as a distraction, not as inspiration for her next music composition. Therefore a simple playback is needed, as though the music were coming from a miniature music box in front.

Perhaps the distinction I am trying to make is a false one; I certainly hope so. It is really a question of how the content is intended to be presented, either by its creator or by its remixer. Every playback of a recording is really the creation of a new song, a mixing of the audio filtered through the quality of the playback system and then mixed with the ambient background noise. A perfect playback system does not remove the need to consider mixing multiple audio sources together or how to selectively mute or master them. But who makes these choices? How do we enable security, e.g. so that a user is not lulled into thinking there is a floor where in reality a sheer cliff exists?

Collaboration has a natural analogy from real space to virtual space; two people in a room whether that room is real or virtual. But the analogy is not true; communications have latency, people are not always there, and you may not even know who or what the other person is. In the end, we are all just bunches of neurons, approximately 100 billion, whereas there are only 7 or 8 billion people. So one could conceivably devote a neuron to each person in the world, and network them together using a brain-computer interface and high-speed networking; such experiments have apparently been done on a small scale with rats (unfortunately I've lost the link; it used to be on Wikipedia) and shown to give a limited form of telepathy; they could coordinate and “swarm” more effectively than they could indiviually. Would this be the “ultimate” in collaboration? Perhaps not; maybe we would need to link more neurons or come up with other tools to communicate. But it seems like a good goal to strive for, in concept if not in implementation.

That, I think, satisfies ubiquity; what interface is more usable than that of your body? It is not so clear that it satisfies expressiveness; perhaps there are concepts that cannot be communicated by thought but must be written down. Would poetry have meaning to beings that feel each other's emotions? Humans do not record their senses like video cameras do; on the contrary, their skill lies in ignoring most of the outside world until it becomes necessary to act. So whatever the interface, there must still be some way to share reality as well as humanity; 3d holographic smell-o-vision, as an old teacher of mine used to say. From looking at a few sites, smell seems a little impractical, but 3d is on its way in commercially and real-time holographic displays are in the this-is-cool-but-how-can-we-mass-produce-it stage, i.e. the technology is there and proven but still too expensive (and big) for the average consumer. Haptics are progressing from vibrations to tactile surfaces, but again, nothing consumer-ready. Matrix-like direct interfaces that override the body's senses still seem to be science fiction, unfortunately; I guess people don't like being cyborgs.