Drag & Drop in the Remotion Player
The Remotion Player supports reacting to mouse events allowing for building interactions on the canvas.
Try to drag and resize the elements below.
General considerations
Pointer events work mostly just like in regular React.
Disable the controls prop to disable any obstructing elements. You can render Playback controls outside of the Player.
The Player might have CSS scale() applied to it.
If you measure elements, you need to divide by the scale obtained by useCurrentScale().
You can pass state update functions via inputProps to the Player.
Alternatively, wrapping the Player in React Context also works.
Example
Let's build the demo above.
The following features have been defined as goals:
- Ability to freely position and resizing the squares with the mouse.
- One item can be selected at a time. Clicking on an empty space will deselect the item.
- Items can not overflow the container, however the blue outlines can overflow the container.
1 Data structure
Create a TypeScript type that describes the shape of your item.
In this example we store an identifier id, start and end frame from and
durationInFrames, the position left, top, width and height and a
color.
item.tstsxexport typeItem = {id : number;durationInFrames : number;from : number;height : number;left : number;top : number;width : number;color : string;};
item.tstsxexport typeItem = {id : number;durationInFrames : number;from : number;height : number;left : number;top : number;width : number;color : string;};
If you like to support different types of items (solid, video, image), see here.
2 Item rendering
Declare a React component that renders the item.
Layer.tsxtsximportReact , {useMemo } from 'react';import {Sequence } from 'remotion';import type {Item } from './item';export constLayer :React .FC <{item :Item ;}> = ({item }) => {conststyle :React .CSSProperties =useMemo (() => {return {backgroundColor :item .color ,position : 'absolute',left :item .left ,top :item .top ,width :item .width ,height :item .height ,};}, [item .color ,item .height ,item .left ,item .top ,item .width ]);return (<Sequence key ={item .id }from ={item .from }durationInFrames ={item .durationInFrames }layout ="none"><div style ={style } /></Sequence >);};
Layer.tsxtsximportReact , {useMemo } from 'react';import {Sequence } from 'remotion';import type {Item } from './item';export constLayer :React .FC <{item :Item ;}> = ({item }) => {conststyle :React .CSSProperties =useMemo (() => {return {backgroundColor :item .color ,position : 'absolute',left :item .left ,top :item .top ,width :item .width ,height :item .height ,};}, [item .color ,item .height ,item .left ,item .top ,item .width ]);return (<Sequence key ={item .id }from ={item .from }durationInFrames ={item .durationInFrames }layout ="none"><div style ={style } /></Sequence >);};
By using a <Sequence> component, the item is only displayed from from until from + durationInFrames.
By adding layout="none", the <div> is mounted as a direct child to the DOM.
3 Outline rendering
Create a React component that renders an outline when an item isselected or hovered.
SelectionOutline.tsxtsximportReact , {useCallback ,useMemo } from 'react';import {useCurrentScale } from 'remotion';import {ResizeHandle } from './ResizeHandle';import type {Item } from './item';export constSelectionOutline :React .FC <{item :Item ;changeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;setSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;selectedItem : number | null;}> = ({item ,changeItem ,setSelectedItem ,selectedItem }) => {constscale =useCurrentScale ();constscaledBorder =Math .ceil (2 /scale );const [hovered ,setHovered ] =React .useState (false);constonMouseEnter =useCallback (() => {setHovered (true);}, []);constonMouseLeave =useCallback (() => {setHovered (false);}, []);constisSelected =item .id ===selectedItem ;conststyle :React .CSSProperties =useMemo (() => {return {width :item .width ,height :item .height ,left :item .left ,top :item .top ,position : 'absolute',outline :hovered ||isSelected ? `${scaledBorder }px solid #0B84F3` :undefined ,userSelect : 'none',touchAction : 'none',};}, [hovered ,item ,scaledBorder ,isSelected ]);conststartDragging =useCallback ((e :PointerEvent |React .MouseEvent ) => {constinitialX =e .clientX ;constinitialY =e .clientY ;constonPointerMove = (pointerMoveEvent :PointerEvent ) => {constoffsetX = (pointerMoveEvent .clientX -initialX ) /scale ;constoffsetY = (pointerMoveEvent .clientY -initialY ) /scale ;changeItem (item .id , (i ) => {return {...i ,left :Math .round (item .left +offsetX ),top :Math .round (item .top +offsetY ),};});};constonPointerUp = () => {window .removeEventListener ('pointermove',onPointerMove );};window .addEventListener ('pointermove',onPointerMove , {passive : true});window .addEventListener ('pointerup',onPointerUp , {once : true,});},[item ,scale ,changeItem ],);constonPointerDown =useCallback ((e :React .MouseEvent ) => {e .stopPropagation ();if (e .button !== 0) {return;}setSelectedItem (item .id );startDragging (e );},[item .id ,setSelectedItem ,startDragging ],);return (<div onPointerDown ={onPointerDown }onPointerEnter ={onMouseEnter }onPointerLeave ={onMouseLeave }style ={style }>{isSelected ? (<><ResizeHandle item ={item }setItem ={changeItem }type ="top-left" /><ResizeHandle item ={item }setItem ={changeItem }type ="top-right" /><ResizeHandle item ={item }setItem ={changeItem }type ="bottom-left" /><ResizeHandle item ={item }setItem ={changeItem }type ="bottom-right" /></>) : null}</div >);};
SelectionOutline.tsxtsximportReact , {useCallback ,useMemo } from 'react';import {useCurrentScale } from 'remotion';import {ResizeHandle } from './ResizeHandle';import type {Item } from './item';export constSelectionOutline :React .FC <{item :Item ;changeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;setSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;selectedItem : number | null;}> = ({item ,changeItem ,setSelectedItem ,selectedItem }) => {constscale =useCurrentScale ();constscaledBorder =Math .ceil (2 /scale );const [hovered ,setHovered ] =React .useState (false);constonMouseEnter =useCallback (() => {setHovered (true);}, []);constonMouseLeave =useCallback (() => {setHovered (false);}, []);constisSelected =item .id ===selectedItem ;conststyle :React .CSSProperties =useMemo (() => {return {width :item .width ,height :item .height ,left :item .left ,top :item .top ,position : 'absolute',outline :hovered ||isSelected ? `${scaledBorder }px solid #0B84F3` :undefined ,userSelect : 'none',touchAction : 'none',};}, [hovered ,item ,scaledBorder ,isSelected ]);conststartDragging =useCallback ((e :PointerEvent |React .MouseEvent ) => {constinitialX =e .clientX ;constinitialY =e .clientY ;constonPointerMove = (pointerMoveEvent :PointerEvent ) => {constoffsetX = (pointerMoveEvent .clientX -initialX ) /scale ;constoffsetY = (pointerMoveEvent .clientY -initialY ) /scale ;changeItem (item .id , (i ) => {return {...i ,left :Math .round (item .left +offsetX ),top :Math .round (item .top +offsetY ),};});};constonPointerUp = () => {window .removeEventListener ('pointermove',onPointerMove );};window .addEventListener ('pointermove',onPointerMove , {passive : true});window .addEventListener ('pointerup',onPointerUp , {once : true,});},[item ,scale ,changeItem ],);constonPointerDown =useCallback ((e :React .MouseEvent ) => {e .stopPropagation ();if (e .button !== 0) {return;}setSelectedItem (item .id );startDragging (e );},[item .id ,setSelectedItem ,startDragging ],);return (<div onPointerDown ={onPointerDown }onPointerEnter ={onMouseEnter }onPointerLeave ={onMouseLeave }style ={style }>{isSelected ? (<><ResizeHandle item ={item }setItem ={changeItem }type ="top-left" /><ResizeHandle item ={item }setItem ={changeItem }type ="top-right" /><ResizeHandle item ={item }setItem ={changeItem }type ="bottom-left" /><ResizeHandle item ={item }setItem ={changeItem }type ="bottom-right" /></>) : null}</div >);};
ResizeHandle.tsxtsximportReact , {useCallback ,useMemo } from 'react';import {useCurrentScale } from 'remotion';import type {Item } from './item';constHANDLE_SIZE = 8;export constResizeHandle :React .FC <{type : 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right';setItem : (itemId : number,updater : (item :Item ) =>Item ) => void;item :Item ;}> = ({type ,setItem ,item }) => {constscale =useCurrentScale ();constsize =Math .round (HANDLE_SIZE /scale );constborderSize = 1 /scale ;constsizeStyle :React .CSSProperties =useMemo (() => {return {position : 'absolute',height :size ,width :size ,backgroundColor : 'white',border : `${borderSize }px solid #0B84F3`,};}, [borderSize ,size ]);constmargin = -size / 2 -borderSize ;conststyle :React .CSSProperties =useMemo (() => {if (type === 'top-left') {return {...sizeStyle ,marginLeft :margin ,marginTop :margin ,cursor : 'nwse-resize',};}if (type === 'top-right') {return {...sizeStyle ,marginTop :margin ,marginRight :margin ,right : 0,cursor : 'nesw-resize',};}if (type === 'bottom-left') {return {...sizeStyle ,marginBottom :margin ,marginLeft :margin ,bottom : 0,cursor : 'nesw-resize',};}if (type === 'bottom-right') {return {...sizeStyle ,marginBottom :margin ,marginRight :margin ,right : 0,bottom : 0,cursor : 'nwse-resize',};}throw newError ('Unknown type: ' +JSON .stringify (type ));}, [margin ,sizeStyle ,type ]);constonPointerDown =useCallback ((e :React .MouseEvent ) => {e .stopPropagation ();constinitialX =e .clientX ;constinitialY =e .clientY ;constonPointerMove = (pointerMoveEvent :PointerEvent ) => {constoffsetX = (pointerMoveEvent .clientX -initialX ) /scale ;constoffsetY = (pointerMoveEvent .clientY -initialY ) /scale ;constisLeft =type === 'top-left' ||type === 'bottom-left';constisTop =type === 'top-left' ||type === 'top-right';setItem (item .id , (i ) => {constnewWidth =item .width + (isLeft ? -offsetX :offsetX );constnewHeight =item .height + (isTop ? -offsetY :offsetY );constnewLeft =item .left + (isLeft ?offsetX : 0);constnewTop =item .top + (isTop ?offsetY : 0);return {...i ,width :Math .max (1,Math .round (newWidth )),height :Math .max (1,Math .round (newHeight )),left :Math .min (item .left +item .width - 1,Math .round (newLeft )),top :Math .min (item .top +item .height - 1,Math .round (newTop )),};});};constonPointerUp = () => {window .removeEventListener ('pointermove',onPointerMove );};window .addEventListener ('pointermove',onPointerMove , {passive : true});window .addEventListener ('pointerup',onPointerUp , {once : true,});},[item ,scale ,setItem ,type ],);return <div onPointerDown ={onPointerDown }style ={style } />;};
ResizeHandle.tsxtsximportReact , {useCallback ,useMemo } from 'react';import {useCurrentScale } from 'remotion';import type {Item } from './item';constHANDLE_SIZE = 8;export constResizeHandle :React .FC <{type : 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right';setItem : (itemId : number,updater : (item :Item ) =>Item ) => void;item :Item ;}> = ({type ,setItem ,item }) => {constscale =useCurrentScale ();constsize =Math .round (HANDLE_SIZE /scale );constborderSize = 1 /scale ;constsizeStyle :React .CSSProperties =useMemo (() => {return {position : 'absolute',height :size ,width :size ,backgroundColor : 'white',border : `${borderSize }px solid #0B84F3`,};}, [borderSize ,size ]);constmargin = -size / 2 -borderSize ;conststyle :React .CSSProperties =useMemo (() => {if (type === 'top-left') {return {...sizeStyle ,marginLeft :margin ,marginTop :margin ,cursor : 'nwse-resize',};}if (type === 'top-right') {return {...sizeStyle ,marginTop :margin ,marginRight :margin ,right : 0,cursor : 'nesw-resize',};}if (type === 'bottom-left') {return {...sizeStyle ,marginBottom :margin ,marginLeft :margin ,bottom : 0,cursor : 'nesw-resize',};}if (type === 'bottom-right') {return {...sizeStyle ,marginBottom :margin ,marginRight :margin ,right : 0,bottom : 0,cursor : 'nwse-resize',};}throw newError ('Unknown type: ' +JSON .stringify (type ));}, [margin ,sizeStyle ,type ]);constonPointerDown =useCallback ((e :React .MouseEvent ) => {e .stopPropagation ();constinitialX =e .clientX ;constinitialY =e .clientY ;constonPointerMove = (pointerMoveEvent :PointerEvent ) => {constoffsetX = (pointerMoveEvent .clientX -initialX ) /scale ;constoffsetY = (pointerMoveEvent .clientY -initialY ) /scale ;constisLeft =type === 'top-left' ||type === 'bottom-left';constisTop =type === 'top-left' ||type === 'top-right';setItem (item .id , (i ) => {constnewWidth =item .width + (isLeft ? -offsetX :offsetX );constnewHeight =item .height + (isTop ? -offsetY :offsetY );constnewLeft =item .left + (isLeft ?offsetX : 0);constnewTop =item .top + (isTop ?offsetY : 0);return {...i ,width :Math .max (1,Math .round (newWidth )),height :Math .max (1,Math .round (newHeight )),left :Math .min (item .left +item .width - 1,Math .round (newLeft )),top :Math .min (item .top +item .height - 1,Math .round (newTop )),};});};constonPointerUp = () => {window .removeEventListener ('pointermove',onPointerMove );};window .addEventListener ('pointermove',onPointerMove , {passive : true});window .addEventListener ('pointerup',onPointerUp , {once : true,});},[item ,scale ,setItem ,type ],);return <div onPointerDown ={onPointerDown }style ={style } />;};
Z-indexing: These elements will later in this tutorial be rendered on top of all layers so that they are able to accept mouse events.
Clicking on one of these elements will select the underlying item.
Scaling: The <Player /> has a CSS scale transform applied to it.
Since we consume the coordinates of the pointer events, we need to divide them by useCurrentScale() to get the correct position.
The thickness of the borders will also be affected by the scale, so we need to divide them by the scale too.
Pointer event hadling: On any pointerdown event, we call e.stopPropagation() prevent the event from bubbling up.
Later, we will treat any event that bubbles up as an click on an empty space and deselect the item.
We also don't do any action if e.button !== 0. This prevents a bug where the item moves after a right click.
We add userSelect: 'none' to disable the native selection and drag behavior of the browser.
We also add touchAction: 'none to disable scrolling the page on mobile while dragging.
4 Sorting outlines
Let's create a component that renders all outlines.
The item which gets rendered as last element will be on top of all other items.
The item that is selected should be rendered on top of all other items because it is the item that is responsive to mouse events.
We render the selected outlines last. This ensures that the resize handles are always on top.
This logic only applies to the outlines, not the actual layers.
SortedOutlines.tsxtsximportReact from 'react';import {Sequence } from 'remotion';import {SelectionOutline } from './SelectionOutline';import type {Item } from './item';constdisplaySelectedItemOnTop = (items :Item [],selectedItem : number | null,):Item [] => {constselectedItems =items .filter ((item ) =>item .id ===selectedItem );constunselectedItems =items .filter ((item ) =>item .id !==selectedItem );return [...unselectedItems , ...selectedItems ];};export constSortedOutlines :React .FC <{items :Item [];selectedItem : number | null;changeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;setSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;}> = ({items ,selectedItem ,changeItem ,setSelectedItem }) => {constitemsToDisplay =React .useMemo (() =>displaySelectedItemOnTop (items ,selectedItem ),[items ,selectedItem ],);returnitemsToDisplay .map ((item ) => {return (<Sequence key ={item .id }from ={item .from }durationInFrames ={item .durationInFrames }layout ="none"><SelectionOutline changeItem ={changeItem }item ={item }setSelectedItem ={setSelectedItem }selectedItem ={selectedItem }/></Sequence >);});};
SortedOutlines.tsxtsximportReact from 'react';import {Sequence } from 'remotion';import {SelectionOutline } from './SelectionOutline';import type {Item } from './item';constdisplaySelectedItemOnTop = (items :Item [],selectedItem : number | null,):Item [] => {constselectedItems =items .filter ((item ) =>item .id ===selectedItem );constunselectedItems =items .filter ((item ) =>item .id !==selectedItem );return [...unselectedItems , ...selectedItems ];};export constSortedOutlines :React .FC <{items :Item [];selectedItem : number | null;changeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;setSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;}> = ({items ,selectedItem ,changeItem ,setSelectedItem }) => {constitemsToDisplay =React .useMemo (() =>displaySelectedItemOnTop (items ,selectedItem ),[items ,selectedItem ],);returnitemsToDisplay .map ((item ) => {return (<Sequence key ={item .id }from ={item .from }durationInFrames ={item .durationInFrames }layout ="none"><SelectionOutline changeItem ={changeItem }item ={item }setSelectedItem ={setSelectedItem }selectedItem ={selectedItem }/></Sequence >);});};
5 Putting it together
Next, we create a main component that renders the items and the outlines.
SortedOutlines.tsxtsximportReact , {useCallback } from 'react';import {AbsoluteFill } from 'remotion';import type {Item } from './item';import {Layer } from './Layer';import {SortedOutlines } from './SortedOutlines';export typeMainProps = {readonlyitems :Item [];readonlysetSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;readonlyselectedItem : number | null;readonlychangeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;};constouter :React .CSSProperties = {backgroundColor : '#eee',};constlayerContainer :React .CSSProperties = {overflow : 'hidden',};export constMain :React .FC <MainProps > = ({items ,setSelectedItem ,selectedItem ,changeItem ,}) => {constonPointerDown =useCallback ((e :React .PointerEvent ) => {if (e .button !== 0) {return;}setSelectedItem (null);},[setSelectedItem ],);return (<AbsoluteFill style ={outer }onPointerDown ={onPointerDown }><AbsoluteFill style ={layerContainer }>{items .map ((item ) => {return <Layer key ={item .id }item ={item } />;})}</AbsoluteFill ><SortedOutlines selectedItem ={selectedItem }items ={items }setSelectedItem ={setSelectedItem }changeItem ={changeItem }/></AbsoluteFill >);};
SortedOutlines.tsxtsximportReact , {useCallback } from 'react';import {AbsoluteFill } from 'remotion';import type {Item } from './item';import {Layer } from './Layer';import {SortedOutlines } from './SortedOutlines';export typeMainProps = {readonlyitems :Item [];readonlysetSelectedItem :React .Dispatch <React .SetStateAction <number | null>>;readonlyselectedItem : number | null;readonlychangeItem : (itemId : number,updater : (item :Item ) =>Item ) => void;};constouter :React .CSSProperties = {backgroundColor : '#eee',};constlayerContainer :React .CSSProperties = {overflow : 'hidden',};export constMain :React .FC <MainProps > = ({items ,setSelectedItem ,selectedItem ,changeItem ,}) => {constonPointerDown =useCallback ((e :React .PointerEvent ) => {if (e .button !== 0) {return;}setSelectedItem (null);},[setSelectedItem ],);return (<AbsoluteFill style ={outer }onPointerDown ={onPointerDown }><AbsoluteFill style ={layerContainer }>{items .map ((item ) => {return <Layer key ={item .id }item ={item } />;})}</AbsoluteFill ><SortedOutlines selectedItem ={selectedItem }items ={items }setSelectedItem ={setSelectedItem }changeItem ={changeItem }/></AbsoluteFill >);};
Z-indexing: We render the layers first and then the outlines. This means the outlines get rendered on top the items.
Overflow behavior: Note that we apply a different overflow behavior to the layers and the outlines.
We hide the overflow of the layers, but show the overflow of the outlines.
This is a UI behavior that is similar to Figma frames.
Click handling: We also listen to pointerdown events on the canvas.
Any event that was not stopped using e.stopPropagation() before will be treated as a click on an empty space and results in deselecting the item.
6 Rendering the Player
We can now render the Main component that we created in the Remotion Player.
Demo.tsxtsximport {Player } from '@remotion/player';importReact , {useCallback ,useMemo ,useState } from 'react';import type {MainProps } from './Main';import {Main } from './Main';import type {Item } from './item';export constDragAndDropDemo :React .FC = () => {const [items ,setItems ] =useState <Item []>([{left : 395,top : 270,width : 540,durationInFrames : 100,from : 0,height : 540,id : 0,color : '#ccc',},{left : 985,top : 270,width : 540,durationInFrames : 100,from : 0,height : 540,id : 1,color : '#ccc',},]);const [selectedItem ,setSelectedItem ] =useState <number | null>(null);constchangeItem =useCallback ((itemId : number,updater : (item :Item ) =>Item ) => {setItems ((oldItems ) => {returnoldItems .map ((item ) => {if (item .id ===itemId ) {returnupdater (item );}returnitem ;});});},[],);constinputProps :MainProps =useMemo (() => {return {items ,setSelectedItem ,changeItem ,selectedItem ,};}, [changeItem ,items ,selectedItem ]);return (<Player style ={{width : '100%',}}component ={Main }compositionHeight ={1080}compositionWidth ={1920}durationInFrames ={300}fps ={30}inputProps ={inputProps }overflowVisible />);};
Demo.tsxtsximport {Player } from '@remotion/player';importReact , {useCallback ,useMemo ,useState } from 'react';import type {MainProps } from './Main';import {Main } from './Main';import type {Item } from './item';export constDragAndDropDemo :React .FC = () => {const [items ,setItems ] =useState <Item []>([{left : 395,top : 270,width : 540,durationInFrames : 100,from : 0,height : 540,id : 0,color : '#ccc',},{left : 985,top : 270,width : 540,durationInFrames : 100,from : 0,height : 540,id : 1,color : '#ccc',},]);const [selectedItem ,setSelectedItem ] =useState <number | null>(null);constchangeItem =useCallback ((itemId : number,updater : (item :Item ) =>Item ) => {setItems ((oldItems ) => {returnoldItems .map ((item ) => {if (item .id ===itemId ) {returnupdater (item );}returnitem ;});});},[],);constinputProps :MainProps =useMemo (() => {return {items ,setSelectedItem ,changeItem ,selectedItem ,};}, [changeItem ,items ,selectedItem ]);return (<Player style ={{width : '100%',}}component ={Main }compositionHeight ={1080}compositionWidth ={1920}durationInFrames ={300}fps ={30}inputProps ={inputProps }overflowVisible />);};
We omit the controls prop to disable the built-in controls.
You can add custom controls outside the Player.
We add the overflowVisible prop to make the outlines visible if they go outside the canvas.
Remember that we've hidden the overflow of the layers itself because it would also not be visible in the final video.
We've now implemented thoughtful drag and drop interactions!