JavaScript (spidermonkey) version

 1fd = File('p.test');
 2fd.open("read",'text');
 3
 4mre = /^__MULTI_TOKEN__\s+(\S+)\s+(.*)\t?\s*(\d+)\s*$/;
 5sre = /^__SINGLE_TOKEN__\s+(\S+)\s*\t?\s*(\d+)\s*$/;
 6
 7var ofd = File('full.txt');
 8ofd.open('write,create,replace','text');
 9first = {}
10
11while (line = fd.readln()) {
12    if (m = line.match(mre)) {
13        first[m[1]] = (first[m[1]] ? first[m[1]] : 0) + parseInt(m[3]);
14        ofd.writeln(m[1] + " " + m[2] + "\t" + m[3]);
15    } else if (m = line.match(sre)) {
16        first[m[1]] = (first[m[1]] ? first[m[1]] : 0) + parseInt(m[2]);
17    } else {
18        print("Unknown: " + line);
19    }
20}
21
22ofd.close()
23
24ofd = File('first.txt');
25ofd.open('write,create,replace','text');
26for (key in first) { 
27    ofd.writeln(key + "\t" + first[key]);
28}
29ofd.close()