To reverse engineer rtmp communication, data stream between client and server has to be captured and analyzed somehow. The best and most professional tool for this is wireshark ( previously ethereal ), it is a free and multi-platformed application.

The other two thing needed is an rtmp-capable server ( red5, wowza, or Adobe's FMS ), and a test application with a simple AMF0 - encoded NetConnection.

Start a new live capture in wireshark with an rtmp port filter ( port == 1935 ), then run the test application in flex, and stop both after a successful connection.

The first data from the client is an 1537 bytes long dump:

0030                           03 01 a8 95 86 00 00 00  r..6r..6........
0040   00 3f 00 cc 00 3d 00 e2 00 4b 00 48 00 e9 00 7e  .?...=...K.H...~
0050   00 97 00 04 00 d5 00 5a 00 23 00 00 00 01 00 76  .......Z.#.....v
0060   00 ef 00 3c 00 6d 00 d2 00 fb 00 b8 00 19 00 6e  ...<.m.........n
0070   00 47 00 74 00 05 00 4a 00 d3 00 70 00 31 00 66  .G.t...J...p.1.f
0080   00 9f 00 ac 00 9d 00 c2 00 ab 00 28 00 49 00 5e  ...........(|>.I.^
0090   00 f7 00 e4 00 35 00 3a 00 83 00 e0 00 61 00 56  .....5.:.....a.V
00a0   00 4f 00 1c 00 cd 00 b2 00 5b 00 98 00 79 00 4e  .O.......[...y.N
00b0   00 a7 00 54 00 65 00 2a 00 33 00 50 00 91 00 46  ...T.e.*.3.P...F
00c0   00 ff 00 8c 00 fd 00 a2 00 0b 00 08 00 a9 00 3e  ...............>
00d0   00 57 00 c4 00 95 00 1a 00 e3 00 c0 00 c1 00 36  .W.............6
00e0   00 af 00 fc 00 2d 00 92 00 bb 00 78 00 d9 00 2e  .....-.....x....
00f0   00 07 00 34 00 c5 00 0a 00 93 00 30 00 f1 00 26  ...4.......0...&
0100   00 5f 00 6c 00 5d 00 82 00 6b 00 e8 00 09 00 1e  ._.l.]...k......
0110   00 b7 00 a4 00 f5 00 fa 00 43 00 a0 00 21 00 16  .........C...!..
0120   00 0f 00 dc 00 8d 00 72 00 1b 00 58 00 39 00 0e  .......r...X.9..
0130   00 67 00 14 00 25 00 ea 00 f3 00 10 00 51 00 06  .g...%.......Q..
0140   00 bf 00 4c 00 bd 00 62 00 cb 00 c8 00 69 00 fe  ...L...b.....i..
0150   00 17 00 84 00 55 00 da 00 a3 00 80 00 81 00 f6  .....U..........
0160   00 6f 00 bc 00 ed 00 52 00 7b 00 38 00 99 00 ee  .o.....R.{.8....
0170   00 c7 00 f4 00 85 00 ca 00 53 00 f0 00 b1 00 e6  .........S......
0180   00 1f 00 2c 00 1d 00 42 00 2b 00 a8 00 c9 00 de  ...,...B.+......
0190   00 77 00 64 00 b5 00 ba 00 03 00 60 00 e1 00 d6  .w.d.......`....
01a0   00 cf 00 9c 00 4d 00 32 00 db 00 18 00 f9 00 ce  .....M.2........
01b0   00 27 00 d4 00 e5 00 aa 00 b3 00 d0 00 11 00 c6  .'..............
01c0   00 7f 00 0c 00 7d 00 22 00 8b 00 88 00 29 00 be  .....}.".....)..
01d0   00 d7 00 44 00 15 00 9a 00 63 00 40 00 41 00 b6  ...D.....c.@.A..
01e0   00 2f 00 7c 00 ad 00 12 00 3b 00 f8 00 59 00 ae  ./.|.....;...Y..
01f0   00 87 00 b4 00 45 00 8a 00 13 00 b0 00 71 00 a6  .....E.......q..
0200   00 df 00 ec 00 dd d8 02 00 eb a8 68 5c 89 03 9e  ...........h\...
0210   00 37 68 24 00 75 30 7a 00 c3 03 20 00 a1 00 96  .7h$.u0z... ....
0220   00 8f 00 5c 00 0d 00 f2 00 9b 00 d8 00 b9 00 8e  ...\............
0230   00 e7 00 94 00 a5 03 6a 00 73 68 90 00 d1 10 86  .......j.sh.....
0240   00 3f 28 cc 5c 3d b8 e2 02 4b a8 48 5c e9 03 7e  .?(.\=...K.H\..~
0250   00 97 68 04 00 d5 30 5a 00 23 03 00 00 01 00 76  ..h...0Z.#.....v
0260   00 ef 00 3c 00 6d 00 d2 00 fb 00 b8 00 19 00 6e  ...<.m.........n
0270   00 47 00 74 00 05 a8 4a 5c d3 00 70 00 31 00 66  .G.t...J\..p.1.f
0280   00 9f 00 ac 00 9d 28 c2 00 ab 00 28 00 49 03 5e  ......(....(.I.^
0290   00 f7 00 e4 00 35 b8 3a 00 83 01 e0 00 61 03 56  .....5.:.....a.V
02a0   00 4f 00 1c 00 cd 00 b2 11 5b 00 98 00 79 08 4e  .O.......[...y.N
02b0   00 a7 00 54 00 65 00 2a 00 33 00 50 00 91 00 46  ...T.e.*.3.P...F
02c0   00 ff 00 8c 00 fd ec a2 64 0b 00 08 00 a9 00 3e  ........d......>
02d0   00 57 73 c4 02 95 00 1a 00 e3 00 c0 00 c1 00 36  .Ws............6
02e0   00 af 00 fc 00 2d 00 92 00 bb 00 78 00 d9 00 2e  .....-.....x....
02f0   00 07 00 34 00 c5 00 0a 00 93 00 30 00 f1 00 26  ...4.......0...&
0300   00 5f 00 6c 00 5d 00 82 00 6b 00 e8 00 09 00 1e  ._.l.]...k......
0310   00 b7 00 a4 00 f5 00 fa 00 43 00 a0 00 21 00 16  .........C...!..
0320   00 0f 00 dc 00 8d 00 72 00 1b 00 58 00 39 00 0e  .......r...X.9..
0330   00 67 00 14 00 25 00 ea 00 f3 00 10 00 51 00 06  .g...%.......Q..
0340   00 bf 00 4c 00 bd 00 62 00 cb 00 c8 00 69 00 fe  ...L...b.....i..
0350   00 17 00 84 00 55 00 da 00 a3 00 80 00 81 00 f6  .....U..........
0360   00 6f 00 bc 00 ed 00 52 00 7b 00 38 00 99 00 ee  .o.....R.{.8....
0370   00 c7 00 f4 00 85 00 ca 00 53 00 f0 00 b1 00 e6  .........S......
0380   00 1f 00 2c 00 1d 00 42 00 2b 00 a8 00 c9 00 de  ...,...B.+......
0390   00 77 00 64 00 b5 00 ba 00 03 00 60 00 e1 00 d6  .w.d.......`....
03a0   00 cf 00 9c 00 4d 00 32 00 db 00 18 00 f9 00 ce  .....M.2........
03b0   00 27 00 d4 00 e5 00 aa 00 b3 00 d0 00 11 00 c6  .'
..............
03c0   00 7f 00 0c 00 7d 00 22 00 8b 00 88 00 29 00 be  .....}.".....)..
03d0   00 d7 00 44 00 15 00 9a 00 63 00 40 00 41 00 b6  ...D.....c.@.A..
03e0   00 2f 00 7c 00 ad 00 12 00 3b 00 f8 00 59 00 ae  ./.|.....;...Y..
03f0   00 87 00 b4 00 45 00 8a 00 13 00 b0 00 71 00 a6  .....E.......q..
0400   00 df 00 ec 00 dd 00 02 00 eb 00 68 00 89 00 9e  ...........h....
0410   00 37 00 24 00 75 00 7a 00 c3 00 20 00 a1 00 96  .7.$.u.z... ....
0420   00 8f 00 5c 00 0d c3 f2 00 9b 38 d8 80 b9 00 8e  ...\......8.....
0430   00 e7 00 94 00 a5 00 6a 00 73 00 90 00 d1 00 86  .......j.s......
0440   00 3f 00 cc 00 3d 00 e2 00 4b 00 48 00 e9 00 7e  .?...=...K.H...~
0450   00 97 00 04 00 d5 00 5a 00 23 00 00 00 01 00 76  .......Z.#.....v
0460   00 ef 00 3c 00 6d 00 d2 00 fb 00 b8 00 19 00 6e  ...<.m.........n
0470   00 47 00 74 00 05 00 4a 87 d3 00 70 00 31 00 66  .G.t...J...p.1.f
0480   00 9f 00 ac 00 9d 00 c2 00 ab 00 28 00 49 00 5e  ...........(.I.^
0490   00 f7 00 e4 00 35 00 3a 00 83 00 e0 00 61 00 56  .....5.:.....a.V
04a0   00 4f 00 1c 00 cd 00 b2 00 5b 00 98 00 79 00 4e  .O.......[...y.N
04b0   00 a7 00 54 00 65 00 2a 00 33 00 50 00 91 00 46  ...T.e.*.3.P...F
04c0   00 ff 00 8c 00 fd d8 a2 00 0b 5c 08 5c a9 03 3e  ..........\.\..>
04d0   00 57 54 c4 00 95 30 1a 00 e3 03 c0 00 c1 00 36  .WT...0........6
04e0   00 af 00 fc 00 2d 00 92 00 bb 98 78 00 d9 00 2e  .....-.....x....
04f0   00 07 00 34 00 c5 f1 0a 02 93 00 30 64 f1 ac 26  ...4.......0d..&
0500   5c 5f c8 6c 5c 5d c6 82 02 6b 07 e8 00 09 01 1e  \_.l\]...k......
0510   00 b7 6c a4 5c f5 10 fa 00 43 34 a0 87 21 00 16  ..l.\....C4..!..
0520   00 0f 00 dc 00 8d 03 72 02 1b 60 58 00 39 4c 0e  .......r..`X.9L.
0530   00 67 c8 14 5c 25 6c ea 5c f3 80 10 5c 51 03 06  .g..\%l.\...\Q..
0540   00 bf 30 4c 00 bd 00 62 00 cb 00 c8 00 69 00 fe  ..0L...b.....i..
0550   00 17 00 84 00 55 00 da 00 a3 00 80 00 81 00 f6  .....U..........
0560   00 6f ec bc 64 ed 00 52 00 7b 00 38 00 99 73 ee  .o..d..R.{.8..s.
0570   02 c7 00 f4 00 85 00 ca 00 53 00 f0 00 b1 00 e6  .........S......
0580   00 1f 00 2c 00 1d 10 42 00 2b 04 a8 00 c9 0c de  ...,...B.+......
0590   00 77 00 64 00 b5 6d ba c7 03 30 60 1e e1 00 d6  .w.d..m...0`....
05a0   0e cf 08 9c 5c 4d de 32 02 db 00 18 87 f9 73 ce  ....\M.2......s.
05b0   02 27 00 d4 64 e5 00 aa 00 b3 00 d0 00 11 b2 c6  .'..d...........
05c0   c7 7f ad 0c 8d 7d 4e 22 00 8b c5 88 88 29 06 be  .....}N"
.....)..
05d0   00 d7 ad 44 8d 15 e8 9a 00 63 00 40 0e 41 ad b6  ...D.....c.@.A..
05e0   8d 2f 4e 7c 00 ad 00 12 1e 3b ad f8 8d 59 30 ae  ./N|.....;...Y0.
05f0   1e 87 48 b4 5c 45 6c 8a c7 13 86 b0 a8 71 00 a6  ..H.\El......q..
0600   00 df e8 ec 00 dd e8 02 00 eb 14 68 0e 89 c5 9e  ...........h....
0610   88 37 06 24 00 75 06 7a 00 c3 00 20 0e a1 30 96  .7.$.u.z... ..0.
0620   1e 8f 78 5c 5c 0d b4 f2 37 9b c5 d8 88 b9 06 8e  ..x\\...7.......
0630   00 e7 e8 94 00 a5 00 6a 00                       .......j.


If you connect to the server a few more times, it appears that only the first byte is constant, the last 1536 bytes are generated randomly.

So, the first part of the handshake is a 0x03 byte followed by 1536 random bytes.

Let's check the server's response for this. It's a 3073 bytes length dump, starting with the familiar 0x03. Substracting this from 3073 is 3072, and that is 2 * 1536. Using red5 as the rtmp server the first 1536 is just simple zero-bytes and the second 1536 seems to be the bytes sent by the player from the first handshake.

It looks like the second part of the handshake is a 0x03 byte followed by 1536 bytes generated by the server followed by 1536 bytes generated by the player.

What's next? The player sends 1860 bytes to the server. The first 1536 bytes are zero bytes, it looks they are the bytes received from the server from the previous step. But what is with the second part of the message?

0630                           03 00 00 00 00 01 36 14  ..............6.
0640   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 3f  .......connect.?
0650   f0 00 00 00 00 00 00 03 00 03 61 70 70 02 00 06  ..........app...
0660   6d 69 6c 67 72 61 00 08 66 6c 61 73 68 56 65 72  milgra..flashVer
0670   02 00 0c 4d 41 43 20 39 2c 30 2c 32 38 2c 30 00  ...MAC 9,0,28,0.
0680   06 73 77 66 55 72 6c 02 00 5d 66 69 6c 65 3a 2f  .swfUrl..]file:/
0690   2f 2f 55 73 65 72 73 2f 6d 69 6c 61 6e 74 6f 74  //Users/milantot
06a0   68 2f 53 74 6f 72 65 2f 44 65 76 65 6c 6f 70 6d  h/Store/Developm
06b0   65 6e 74 2f 57 6f 72 6b 73 70 61 63 65 2f 4a 61  ent/Workspace/Ja
06c0   73 6d 69 6e c3 53 6f 75 72 63 65 2f 62 69 6e 2f  smin.Source/bin/
06d0   54 65 73 74 43 6f 6e 6e 65 63 74 69 6f 6e 2d 64  TestConnection-d
06e0   65 62 75 67 2e 73 77 66 00 05 74 63 55 72 6c 02  ebug.swf..tcUrl.
06f0   00 17 72 74 6d 70 3a 2f 2f 6c 6f 63 61 6c 68 6f  ..rtmp://localho
0700   73 74 2f 6d 69 6c 67 72 61 00 04 66 70 61 64 01  st/milgra..fpad.
0710   00 00 0b 61 75 64 69 6f 43 6f 64 65 63 73 00 40  ...audioCodecs.@
0720   83 38 00 00 00 00 00 00 0b 76 69 64 65 6f 43 6f  .8.......videoCo
0730   64 65 63 73 00 40 5f 00 00 00 00 00 00 00 0d 76  decs.@_........v
0740   69 64 65 6f 46 c3 75 6e 63 74 69 6f 6e 00 3f f0  ideoF.unction.?.
0750   00 00 00 00 00 00 00 07 70 61 67 65 55 72 6c 06  ........pageUrl.
0760   00 0e 6f 62 6a 65 63 74 45 6e 63 6f 64 69 6e 67  ..objectEncoding
0770   00 00 00 00 00 00 00 00 00 00 00 09              ............


Well, it looks like this is our first rtmp packet! But  how can we decode this mess? Do you know Smashing Pumpkins? Yes? Then try, try, try! :D

The most boring part comes: from the test application invoke function calls with different identifiers, without arguments, with arguments, with every type of data as argument, and check and analyze the resulting dump. This is my "test dump" :

nc.call( "connect" , null );

0030                           03 00 00 58 00 00 14 14          ...X....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05                          ........

nc.call( "connect" , null );

0030                           03 00 00 09 00 00 14 14          ........
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05                          ........

nc.call( "connect" , null , null );

0030                           03 00 00 36 00 00 15 14          ...6....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 05                       .........

nc.call( "functioncall" , null );

0030                           03 00 00 3f 00 00 19 14          ...?....
0040   00 00 00 00 02 00 0c 66 75 6e 63 74 69 6f 6e 63  .......functionc
0050   61 6c 6c 00 00 00 00 00 00 00 00 00 05           all..........

nc.call( "aaaaaaaaaaaaaaaaaaaa" , null );

0030                           03 00 00 28 00 00 21 14          ...(|>..!.
0040   00 00 00 00 02 00 14 61 61 61 61 61 61 61 61 61  .......aaaaaaaaa
0050   61 61 61 61 61 61 61 61 61 61 61 00 00 00 00 00  aaaaaaaaaaa.....
0060   00 00 00 00 05                                   .....


nc.call( "functioncall" , null , null );

0030                           43 00 00 32 00 00 1a 14          C..2....
0040   02 00 0c 66 75 6e 63 74 69 6f 6e 63 61 6c 6c 00  ...functioncall.
0050   00 00 00 00 00 00 00 00 05 05                    ..........


nc.call( "connect" , null , "connect" );

0030                           03 00 00 33 00 00 1e 14          ...3....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 02 00 07 63 6f 6e 6e 65  ...........conne
0060   63 74                                            ct

nc.call( "connect" , null , new Object( ) );

0030                           03 00 00 66 00 00 18 14          ...f....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 03 00 00 09              ............

var o:Object = new Object( ); o.property = "connect";

0030                           03 00 00 2d 00 00 2c 14          ...-..,.
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 03 00 08 70 72 6f 70 65  ...........prope
0060   72 74 79 02 00 07 63 6f 6e 6e 65 63 74 00 00 09  rty...connect...

nc.call( "connect" , null , { property : "connect" } );

0030                           43 00 00 d9 00 00 2c 14          C.....,.
0040   02 00 07 63 6f 6e 6e 65 63 74 00 00 00 00 00 00  ...connect......
0050   00 00 00 05 03 00 08 70 72 6f 70 65 72 74 79 02  .......property.
0060   00 07 63 6f 6e 6e 65 63 74 00 00 09              ..connect...

nc.call( "connect" , null , { property : "connect" , attribute : "functioncall" } );

0030                           43 00 00 44 00 00 46 14          C..D..F.
0040   02 00 07 63 6f 6e 6e 65 63 74 00 00 00 00 00 00  ...connect......
0050   00 00 00 05 03 00 09 61 74 74 72 69 62 75 74 65  .......attribute
0060   02 00 0c 66 75 6e 63 74 69 6f 6e 63 61 6c 6c 00  ...functioncall.
0070   08 70 72 6f 70 65 72 74 79 02 00 07 63 6f 6e 6e  .property...conn
0080   65 63 74 00 00 09                                ect...

var o:Object = new Object( ); o.property = "connect"; o.attribute = "functioncall";
nc.call( "connect" , null , o );

0030                           03 00 00 46 00 00 46 14          ...F..F.
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 03 00 09 61 74 74 72 69  ...........attri
0060   62 75 74 65 02 00 0c 66 75 6e 63 74 69 6f 6e 63  bute...functionc
0070   61 6c 6c 00 08 70 72 6f 70 65 72 74 79 02 00 07  all..property...
0080   63 6f 6e 6e 65 63 74 00 00 09                    connect...

nc.call( "connect" , null , 0 );

0030                           03 00 00 3b 00 00 1d 14          ...;....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00  ................
0060   00                                               .

nc.call( "connect" , null , 1 );

0030                           03 00 00 3d 00 00 1d 14          ...=....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 3f f0 00 00 00 00 00  .........?......
0060   00                                               .

nc.call( "connect" , null , 2 );

0030                           43 00 00 35 00 00 1d 14          C..5....
0040   02 00 07 63 6f 6e 6e 65 63 74 00 00 00 00 00 00  ...connect......
0050   00 00 00 05 00 40 00 00 00 00 00 00 00           .....@.......

nc.call( "connect" , null , 3 );

0030                           03 00 00 29 00 00 1d 14          ...)....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 40 08 00 00 00 00 00  .........@......
0060   00                                               .

nc.call( "connect" , null , 4 );

0030                           03 00 00 38 00 00 1d 14          ...8....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 40 10 00 00 00 00 00  .........@......
0060   00                                               .

nc.call( "connect" , null , 5 );

0030                           03 00 00 34 00 00 1d 14          ...4....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 40 14 00 00 00 00 00  .........@......
0060   00                                               .

nc.call( "connect" , null , 6 );

0030                           03 00 00 41 00 00 1d 14          ...A....
0040   00 00 00 00 02 00 07 63 6f 6e 6e 65 63 74 00 00  .......connect..
0050   00 00 00 00 00 00 00 05 00 40 18 00 00 00 00 00  .........@......
0060   00                                               .