FreeWRL / FreeX3D 4.3.0
MPEG_Utils.c
1/*
2
3
4???
5
6*/
7
8/****************************************************************************
9 This file is part of the FreeWRL/FreeX3D Distribution.
10
11 Copyright 2009 CRC Canada. (http://www.crc.gc.ca)
12
13 FreeWRL/FreeX3D is free software: you can redistribute it and/or modify
14 it under the terms of the GNU Lesser Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
17
18 FreeWRL/FreeX3D is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with FreeWRL/FreeX3D. If not, see <http://www.gnu.org/licenses/>.
25****************************************************************************/
26
27
28/* NOTE: we have to re-implement the loading of movie textures; the code in here was a decade old and did not
29keep up with "the times". Check for ifdef HAVE_TO_REIMPLEMENT_MOVIETEXTURES in the code */
30
31/* July 2016 note:
32 http://www.web3d.org/documents/specifications/19775-1/V3.3/Part01/components/texturing.html#MovieTexture
33 - specs say to support MPEG1
34 - online says all patents have expired for MPEG1 and its Level II audio
35 http://www.web3d.org/x3d/content/examples/ConformanceNist/
36 - see Movie texture example with VTS.mpg
37
38 A. mpeg1 capable library plumbing
39 There are still patents and licensing issues with recent mp4 and even mpeg-2 I've heard.
40 MPEG1 - any patents have expired
41 Goal: a generic interface we can use to wrap 3 implementations:
42 1. stub
43 2. option: old-fashioned cross-platform mpeg1 c code, as fallback/default
44 3. platform-supplied/platform-specific video API/libraries
45 and to use both audio and video streams, with audio compatible with SoundSource node api needs.
46
47 Links >
48 2. old fashioned mpeg1
49 Mpeg-1 libs / code
50 2.1 reference implementation
51 https://en.wikipedia.org/wiki/MPEG-1
52 See source code link at bottom, reference implementation
53 which has an ISO license which dug9 reads to mean:
54 - use for simulation of what electronic devices will do, presumably by electronics companies
55 - (but not intendended / foreseen as a software library product, uncertain if allowed)
56 it includes audio and video
57 2.2 Berkley mpeg_play derivitives
58 a) berkley_brown
59 original freewrl implementation (taken out in 2009), has both berkley and brown U listed in license section
60 b) berkley_gerg
61 http://www.gerg.ca/software/mpeglib/
62 Mpeg1 - hack of berkley mpeg1 code, no separate license on hacks show
63 c) berkley_nabg
64 http://www.uow.edu.au/~nabg/MPEG/mpeg1.html
65 Mpeg1 explanation with hacked Berkley code and license: berkley + do-what-you-want-with-hacked-code
66 2.3 ffmpeg.org
67 LGPL by default (can add in GPL parts, we don't need)
68 but may include patented algorithms for post-MPEG1-Audio_Level_II
69 but has a way to limit codecs available at runtime:
70 instead of load_all() you would somehow say which ones to load?? see dranger tutorial about tut5
71 and if so and you limit codecs to MPEG1 with Level II audio, then
72 ffplay.c and tutorial http://dranger.com/ffmpeg/ can help,
73 substituting freewrl's audio lib's API, pthreads, and our openGL plumbing for SDL
74 might be able to #define some pthread for SDL thread functions in ffplay.c
75 - except don't assume freewrl's pthreads is complete: no cancel, its an emulated lib on some platforms
76 https://github.com/FFmpeg/FFmpeg
77 https://github.com/FFmpeg/FFmpeg/blob/master/ffplay.c
78 line interesting function
79 3352 event_loop() - play, pause, rewind user functions
80 3000 infinite_buffer real_time -in theory could pull from url instead of file
81 2660 forcing codec by name - if blocking patented MPEG2+ / sticking to MPEG1+LevelII audio
82 2400 audio_decode_frame - somwhere we need to get the audio PCM decompressed buffer, so we can pass to our audio API
83 1808 get_video_frame - somewhere we need to get the current frame, perhaps from do_MovieTextureTick() we would get closest-to-current frame and freeze it for opengl
84 1506 video_refresh called to display each frame
85
86
87 3. Platform supplied
88 3.1 windows > directX audio and video (I have the dx audio working in one app)
89 3.2 android >
90 3.3 linux desktop >
91
92 B. freewrl texture and sound plumbing
93 MAJOR DESIGN OPTIONS:
94 I. Process video frame into mipmapped opengl texture:
95 a) on loading:, pre-process entire video into mipmapped opengl textures, one per video frame (2009 implementation)
96 Disadvantage: 30fps x 10 seconds is 300 textures - a lot to store, and prepare if unneeded
97 x a lot of opengl textures needed - exhaustion/limits?
98 Benefit: faster per frame when playing
99 b) on-the-fly during play: in a separate thread, process video frames to replace single opengl texture
100 Benefit: vs c: mpeg decompression: successive frames require previous frame decoded, so state is managed
101 - vs a,c: thread can do its own throttling
102 - vs a: storage is just the decompression sequence
103 Disadvantage: single core will be doing a lot of un-needed mipmapping
104 x thread collisions - texture being replaced in one thread and drawn in another?
105 x stereo vision > left and right eye might see different texture
106 c) on-the-fly in do_tick (once per render frame), interpolate closest frame based on time, replace single opengl texture
107 Benefit: vs a,b: no unnecessary frames interpolated, no unnecessary mipmapping in any thread, just the frame needed
108 vs. b: same frame appears in left/right stereo, no timing weirdness
109 vs. a: storage is just the decompression sequence
110 d) combo of b) and c): separate thread prepares small set of raw frames,
111 do_MovieTextureTick() picks one or asks for one and opengl-izes it
112 II. Support streaming?
113 a) Continuous Streaming video from url or
114 b) just finite local file
115 SIMPLIFYING DECISION: b) finite local file
116 III. Separate thread for decoding / interpolating frames?
117 a) or in rendering thread (any throttling problems?)
118 b) new separate thread (but what about mipmapping and opengizing interpolated frame?)
119 c) (somehow) use texture thread which is currently parked/unused once images are mipmapped
120 - but currently triggered during image file loading process
121 - could set a flag to an earlier state and re-submit?
122 SIMPLIFYING DECISION: depends on implementation ie what libs you get and how easy it is
123
124 A few facts / details:
125 input media: MPEG1 contains Audio and/or Video
126 Nodes: which we want to use to support SoundSource and/or Texture2D
127 Texture2D: shows one frame at a time in opengl
128 SoundSource: used as / like an AudioClip for Sound node. AudioClip has its own private thread
129 It doesn't make sense to load a movietexture 2x if using for both texture and sound.
130 - you would DEF for one use, and USE for the other
131 - then when you play, you play both USEs at the same time so audio and video are synced
132 Sound is handled per-node.
133 Textures have an intermediary texturetableindexstruct
134
135 2003 - 2009 freewrl movietexture:
136 - on load: decoded and created an opengl texture for each movie frame
137 - used berkley mpeg aka berkley-brown
138
139
140 Proposed freewrl plumbing:
141
142 1. for texture rendering, MovieTexture works like ImageTexture on each render_hier frame,
143 with a single opengl texture number
144 2. leave it to the library-specifics to decide if
145 a) decode-on-load
146 b) decode in a separate thread, anticipatory/queue
147 c) decode-on-demand
148 3. the pause, stop, play, rewind interface is usable as SoundSource like Audioclip, analogous to AudioClip
149 4. perl > make AudioClip and MovieTexture fields in same order, so MovieTexture can be up-caste to AudioClip DONE
150
151 top level interface:
152 X3DMovieTexture._movie; an opaque pointer that will hold a malloced struct representing
153 the movie container and streams; different implementations will be different
154 movie_load() - like loadTextures.c image loaders - takes local path and gets things started
155 puts intial load into ._movie of the requesting node ie res->(wheretoplacedata,offset) = (MovieTexture,_movie)
156 do_MovieTextureTick()
157 in senseInterp.c, once per frame (so stereo uses same movie frame for left/right)
158 could ask for closest frame based on movie time
159 node->tti->texdata = getClosestMovieFrame(movietime)
160 a) decode-on-load would have the frame ready in a list
161 b) multi-thread anticipatory decode would have a private queue/list of decoded frames,
162 and get the closest one, and discard stale frames, and restart decode queue to fill up
163 queue again
164 c) decode-on-demand would decode on demand
165 Texture2D(,,,node->tti->opeglTexture,,,node->tti->texdata) //reset texture data
166
167 loadstatus_MovieTexture(struct X3D_MovieTexture *node) - loadsensor can check if file loaded
168 loadstatus_AudioClip(struct X3D_AudioClip *node) - loadsensor can check if file loaded
169 locateAudioSource (struct X3D_AudioClip *node) - will work for MovieTexture
170
171 search code for MovieTexture, resm_movie to see all hits
172
173
174 MPEG_Utils_berkley.c Nov 15, 2016: compiled but bombs on loading even simple vts.mpg
175 MPEG_Utils_libmpeg2.c Nove 15, 2016: not attempted to implement
176 x undocumented code
177 x GPL
178 x uses callback for frames
179 x no audio channel
180 - but small test program mpeg2dec does run on windows
181 - code seems lite / compact
182*/
183#include <config.h>
184#include <system.h>
185#include <system_threads.h>
186#include <display.h>
187#include <internal.h>
188#include "vrml_parser/CRoutes.h"
189#include "vrml_parser/Structs.h"
190#include "main/ProdCon.h"
191#include "../opengl/OpenGL_Utils.h"
192#include "../opengl/Textures.h"
193#include "../opengl/LoadTextures.h"
194#include "../scenegraph/Component_CubeMapTexturing.h"
195
196#include <list.h>
197#include <io_files.h>
198#include <io_http.h>
199
200#include <threads.h>
201
202#include <libFreeWRL.h>
203
204//put your choice in your config.h (or windows preprocessor directives):
205//#define MOVIETEXTURE_STUB 1 //default
206//#define MOVIETEXTURE_BERKLEYBROWN 1
207//#define MOVIETEXTURE_FFMPEG 1
208//#define MOVIETEXTURE_LIBMPEG2 1
209
210//Option A.
211// movie_load - load as BLOB using standard FILE2BLOB in io_files.c retval = resource_load(res); //FILE2BLOB
212// parse_movie - converts BLOB to sound and video parts, returns parts
213//Option B.
214// movie_load - parse movie as loading
215// parse_movie - return movie parts
216
217#ifdef MOVIETEXTURE_BERKLEYBROWN
218#include "MPEG_Utils_berkley.c"
219#elif MOVIETEXTURE_FFMPEG
220//#include "MPEG_Utils_ffmpeg.c"
221int movie_load_from_file(char *fname, void **opaque);
222double movie_get_duration(void *opaque);
223unsigned char *movie_get_frame_by_fraction(void *opaque, float fraction, int *width, int *height, int *nchan);
224unsigned char * movie_get_audio_PCM_buffer(void *opaque,int *freq, int *channels, int *size, int *bits);
225
226#ifdef HAVE_OPENAL
227#include <AL/al.h>
228#include <AL/alc.h>
229#include <AL/alext.h>
230#ifdef HAVE_ALUT
231#include <AL/alut.h>
232#endif //HAVE_ALUT
233#endif //HAVE_OPENAL
234
235//BufferData * alutBufferDataConstruct (ALvoid *data, size_t length, ALint numChannels,
236// ALint bitsPerSample, ALfloat sampleFrequency);
237
238#define LOAD_STABLE 10 //from Component_Sound.c
239#elif MOVIETEXTURE_LIBMPEG2
240#endif
241
242bool movie_load(resource_item_t *res){
243 bool retval;
244 // see io_files.c for call place
245 //Option A: just load blob for later
246 // retval = resource_load(res); //FILE2BLOB
247 //Option B:
248 // parse during load
249 // copied from imagery_load - but TEX_READ flag will be wrong for movie
250 //int textureNumber;
251 //struct textureTableIndexStruct *entry; // = res->whereToPlaceData;
252 //textureNumber = res->textureNumber;
253 //if(res->status == ress_downloaded){
254 // entry = getTableIndex(textureNumber);
255 // if(entry)
256 // if (movie_load_from_file(entry, res->actual_file)) {
257 // entry->status = TEX_READ; // tell the texture thread to convert data to OpenGL-format
258 // res->status = ress_loaded;
259 // retval = TRUE;
260 // return retval;
261 // }
262 //}
263 //res->status = ress_not_loaded;
264 retval = FALSE;
265
266#ifdef MOVIETEXTURE_STUB
267 res->status = ress_loaded;
268 retval = TRUE;
269#elif MOVIETEXTURE_BERKLEYBROWN
270 {
271 int x,y,depth,frameCount;
272 char *ptr;
273 ptr=NULL;
274 //H: this returns something like a volume image, with slices packed into ptr, and z=frameCount, nchannels = depth.
275 //Q: what's the 'normal' frame rate? should that be returned too, or is there a standard/default?
276 //Nov 15, 2016: bombs on small test file vts.mpg
277 mpg_main(res->actual_file, &x,&y,&depth,&frameCount,&ptr);
278 #ifdef TEXVERBOSE
279 printf ("have x %d y %d depth %d frameCount %d ptr %d\n",x,y,depth,frameCount,ptr);
280 #endif
281 // store_tex_info(loadThisTexture, depth, x, y, ptr,depth==4);
282
283 // and, manually put the frameCount in.
284 //res->frames = frameCount;
285 }
286
287#elif MOVIETEXTURE_FFMPEG
288 {
289 void *opaque;
290 int loaded;
291 loaded = movie_load_from_file(res->actual_file,&opaque);
292 retval = loaded > -1 ? TRUE : FALSE;
293 if(loaded){
294 int freq,channels,size,bits;
295 unsigned char * pcmbuf;
296 struct X3D_MovieTexture *node;
297
298 res->status = ress_loaded;
299 res->complete = TRUE;
300 res->status = ress_parsed; //we'll skip the parse_movie/load_from_blob handler
301
302 node = (struct X3D_MovieTexture *) res->whereToPlaceData;
303 //AUDIO AND/OR VIDEO CHANNELS?
304 node->duration_changed = movie_get_duration(opaque);
305 node->__fw_movie = opaque;
306 node->__loadstatus = LOAD_STABLE;
307 //VIDEO CHANNEL?
308 //double totalframes = node->duration_changed * 30.0;
309 node->speed = 1.0; //1 means normal speed 30.0 / totalframes; //in fractions per second = speed in frames/second / totalframes
310 MARK_EVENT (X3D_NODE(node), offsetof(struct X3D_MovieTexture, duration_changed));
311 //AUDIO CHANNEL?
312 //node->__sourceNumber = parse_movie(node,buffer,len); //__sourceNumber will be openAL buffer number
313 pcmbuf = movie_get_audio_PCM_buffer(opaque,&freq,&channels,&size,&bits);
314 if(pcmbuf){
315 //MPEG1 level1,2 are compressed audio
316 //decoders generally deliver so called PCM pulse code modulated buffers
317 //and that's what audio drivers on computers normally take
318 //and same with the APIs that wrap the hardware drivers ie openAL API
319 printf("audio freq %d channels %d size %d bits per channel %d\n",freq,channels,size,bits);
320#ifdef HAVE_LIBSOUND
321 node->__sourceNumber = libsound_createBusFromPCM(pcmbuf, bits, channels, size, freq);
322#else //HAVE_LIBSOUND
323 #ifdef HAVE_OPENAL
324 // http://open-activewrl.sourceforge.net/data/OpenAL_PGuide.pdf
325 // page 6
326 {
327 int format;
328 ALuint albuffer;
329 static int once = 0;
330 if(!once){
331 #ifdef HAVE_ALUT
332 //alutInit(0, NULL); // Initialize OpenAL
333 if (!alutInitWithoutContext(NULL, NULL))
334 ConsoleMessage("ALUT init failed\n");
335 #endif //HAVE_ALUT
336 alGetError(); // Clear Error Code
337 //SoundEngineInit();
338 once = 1;
339 }
340
341 alGenBuffers(1, &albuffer);
342 //al.h
343 //#define AL_FORMAT_MONO8 0x1100
344 //#define AL_FORMAT_MONO16 0x1101
345 //#define AL_FORMAT_STEREO8 0x1102
346 //#define AL_FORMAT_STEREO16 0x1103
347 //if(bits == 8)
348 // format = AL_FORMAT_MONO8;
349 //else
350 // format = AL_FORMAT_MONO16;
351 //if(channels == 2)
352 // if(bits == 8)
353 // format = AL_FORMAT_STEREO8;
354 // else
355 // format = AL_FORMAT_STEREO16;
356 format = 0;
357 switch(bits){
358 case 8:
359 format = AL_FORMAT_MONO8;
360 if(channels == 2)
361 format = AL_FORMAT_STEREO8;
362 break;
363 case 16:
364 format = AL_FORMAT_MONO16;
365 if (channels == 2)
366 format = AL_FORMAT_STEREO16;
367 break;
368 case 32:
369 #ifdef AL_EXT_float32
370 format = AL_FORMAT_MONO_FLOAT32;
371 if (channels == 2)
372 format = AL_FORMAT_STEREO_FLOAT32;
373 break;
374 #endif
375 default:
376 break;
377 }
378 if(format > 0){
379 //this is a complex function that tries to figure out if its float, int PCM etc
380 alBufferData(albuffer,format,pcmbuf,size,freq);
381 //BufferData * bdata = _alutBufferDataConstruct( pcmbuf,size,channels,bits, freq);
382
383 node->__sourceNumber = albuffer;
384 }
385 }
386 #endif //HAVE_OPENAL
387#endif //HAVE_LIBSOUND
388 }
389 }
390
391 printf("opqaue = %p, loaded=%d \n",opaque,res->status);
392 }
393#elif MOVIETEXTURE_LIBMPEG2
394#endif
395 return retval;
396}
397int parse_audioclip(struct X3D_AudioClip *node,char *bbuffer, int len);
398int parse_movie(struct X3D_MovieTexture *node, char *buffer,int len){
399 //Option B - parse blob
400 //if your movie api will take a blob, you can call it from here to parse
401 //convert BLOB (binary large object) into video and audio structures
402 //Option A and B - return audio and video parts
403 int audio_sourcenumber;
404 audio_sourcenumber = -1; //BADAUDIOSOURCE
405 //MPEG1 level1,2 are compressed audio
406 //decoders generally deliver so called PCM pulse code modulated buffers
407 //and that's what audio drivers on computers normally take
408 //and same with the APIs that wrap the hardware drivers ie openAL API
409#ifdef MOVIETEXTURE_STUB
410#elif MOVIETEXTURE_BERKLEYBROWN
411#elif MOVIETEXTURE_FFMPEG
412#elif MOVIETEXTURE_LIBMPEG2
413#endif
414 return audio_sourcenumber;
415}
416double compute_duration(int ibuffer);
417
418bool process_res_movie(resource_item_t *res){
419 // METHOD_LOAD_ON_DEMAND
420 //you'll get in here if you didn't (completely) handle movie_load from file
421 //
422 //s_list_t *l;
423 openned_file_t *of;
424 char *buffer;
425 int len;
426 struct X3D_MovieTexture *node;
427
428 buffer = NULL;
429 len = 0;
430 switch (res->type) {
431 case rest_invalid:
432 return FALSE;
433 break;
434
435 case rest_string:
436 buffer = res->URLrequest;
437 break;
438 case rest_url:
439 case rest_file:
440 case rest_multi:
441 of = res->openned_files;
442 if (!of) {
443 /* error */
444 return FALSE;
445 }
446
447 buffer = of->fileData;
448 len = of->fileDataSize;
449 break;
450 }
451
452 node = (struct X3D_MovieTexture *) res->whereToPlaceData;
453 //node->__FILEBLOB = buffer;
454 node->__sourceNumber = parse_movie(node,buffer,len); //__sourceNumber will be openAL buffer number
455 if(node->__sourceNumber > -1) {
456 node->duration_changed = compute_duration(node->__sourceNumber);
457 MARK_EVENT (X3D_NODE(node), offsetof(struct X3D_MovieTexture, duration_changed));
458 return TRUE;
459 }
460 return FALSE;
461}
462
463
464// - still needed ? don't know depends on implementation
465//void getMovieTextureOpenGLFrames(int *highest, int *lowest,int myIndex) {
466// textureTableIndexStruct_s *ti;
467//
469// printf ("getMovieTextureOpenGLFrames, myIndex is ZERL\n");
470// *highest=0; *lowest=0;
471// } else {
472//*/
473// *highest=0; *lowest=0;
474//
475// #ifdef TEXVERBOSE
476// printf ("in getMovieTextureOpenGLFrames, calling getTableIndex\n");
477// #endif
478//
479// ti = getTableIndex(myIndex);
480//
482// if (ti->OpenGLTexture != TEXTURE_INVALID) {
483// *lowest = ti->OpenGLTexture;
484// *highest = 0;
486// }
488//}
489
490unsigned char *movietexture_get_frame_by_fraction(struct X3D_Node* node, float fraction, int *width, int *height, int *nchan){
491 unsigned char* retval = NULL;
492 if(node && node->_nodeType == NODE_MovieTexture){
493 struct X3D_MovieTexture *movietexture = (struct X3D_MovieTexture *)node;
494#ifdef MOVIETEXTURE_STUB
495#elif MOVIETEXTURE_BERKLEYBROWN
496#elif MOVIETEXTURE_FFMPEG
497 retval = movie_get_frame_by_fraction(movietexture->__fw_movie,fraction,width,height,nchan);
498#elif MOVIETEXTURE_LIBMPEG2
499#endif
500 }
501 return retval;
502}