MIDI Keyboard Controlling Blender Face

After a couple weeks of on and off work, this is what I have:

I used the mido python library to get the input keys, then made the keys control driver objects with bpy that controlled the face with shape keys.

The audio is from Garage Band, and I used the internal viewport renderer with matcap

Here is a code framework for anyone who wants to use this method (you will need mido and rtmidi in your Blender python):

import mido
import bpy
from time import sleep
mido.set_backend('mido.backends.rtmidi')


port = mido.open_input() #<-- insert name of MIDI input (get list of names of MIDI inputs with mido.get_input_names())

frames = 20000 #duration of running code

beingPressed = [] #contains the keys being pressed in each frame


for frame in range(frames): #for each frame

    for msg in port.iter_pending():  #Update beingPressed
        if msg.velocity == 0:  #if velocity is 0, that means the key has just stopped being pressed
            beingPressed.remove(msg.note) 
        else:  #if the velocity isn't 0, then that means the key just started being pressed
            beingPressed.append(msg.note)

    #put whatever code you want here to do something each frame using beingPressed
    
    #draw the frame
    bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
    sleep(#insert 1/(framerate))



I got the mido and rtmidi files into Blender by using pip3 and brew in terminal to install them onto my system python and then transferred the files from /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages to blender.app/Contents/Resources/2.77/python/lib/python3.5/site-packages
I’m sure there’s a better way to get python libraries into blender, but this worked for me

mido documentation: https://mido.readthedocs.io/en/latest/ports.html

Here’s my code, but it is very specific to my setup (here’s my blend file: MIDI24.blend (2.22 MB)), so it’s probably not very useful:

import mido
import bpy
from time import sleep
mido.set_backend('mido.backends.rtmidi')


port = mido.open_input(u'KeyRig 49')


frames = 20000
controlSpeed = .08


beingPressed = [] #contains the keys being pressed each frame


#Face Drivers change the character
faceDrivers = [bpy.data.objects['Face1'], bpy.data.objects['Face2'], bpy.data.objects['Face3'], bpy.data.objects['Face4'], bpy.data.objects['Face5'], bpy.data.objects['Face6'], bpy.data.objects['Face7'], bpy.data.objects['Face8']]
currentFace = 0


#Driver information format: [Driver obj, increase key, decrease key]
JawUD = [bpy.data.objects['JawUD'],36,38] 
CheekPuffUD = [bpy.data.objects['CheekPuffUD'],37,39] 
JawLR = [bpy.data.objects['JawLR'],41,40]
TopLipUD = [bpy.data.objects['TopLipUD'],43,45]
BotLipUD = [bpy.data.objects['BotLipUD'],47,48]
LipPuckerUD = [bpy.data.objects['LipPuckerUD'],49,51] 
LeftLipUD = [bpy.data.objects['LeftLipUD'],50,52]
LeftLipLR = [bpy.data.objects['LeftLipLR'],55,53]
RightLipUD = [bpy.data.objects['RightLipUD'],57,59]
RightLipLR = [bpy.data.objects['RightLipLR'],62,60]
NoseSniffUD = [bpy.data.objects['NoseSniffUD'],61,63] 
LeftBrowInUD = [bpy.data.objects['LeftBrowInUD'],64,65]
LeftBrowOutUD = [bpy.data.objects['LeftBrowOutUD'],67,69]
RightBrowInUD = [bpy.data.objects['RightBrowInUD'],71,72]
EyeDilateUD = [bpy.data.objects['EyeDilateUD'],73,75] 
RightBrowOutUD = [bpy.data.objects['RightBrowOutUD'],74,76]
LeftEyeLidUD = [bpy.data.objects['LeftEyeLidUD'],77,79]
RightEyeLidUD = [bpy.data.objects['RightEyeLidUD'],81,83]


drivers = [JawUD, CheekPuffUD, JawLR, TopLipUD, BotLipUD, LipPuckerUD, LeftLipUD, LeftLipLR, RightLipUD, RightLipUD, RightLipLR, NoseSniffUD, LeftBrowInUD, LeftBrowOutUD, RightBrowInUD, EyeDilateUD, RightBrowOutUD, LeftEyeLidUD, RightEyeLidUD]


#pose keys set the face back to a preset face pose
#pose keys format: note number: 
[list of locations for each driver]
poseKeys = {
'82':[1, 36, 12, -27, 25, 0, 19, -4, 36, 36, -13, 0, 12, -1, 15, 0, -11, -8, -19], 
'80':[-36, -6, -2, 37, -36, 2, 36, -31, 33, 33, 35, -19, 44, 5, 33, -2, 10, 9, 14], 
'78':[11, 0, 2, -38, 2, 0, 8, 7, 3, 3, -33, 0, 20, -4, 16, 0, 2, 9, 6], 
'70':[-33, -16, 22, 26, -35, -2, -8, 40, -15, -15, -39, -36, -1, -34, -19, -11, -34, -3, 0], 
'68':[41, -9, -16, 2, 45, 13, 4, 36, -5, -5, 18, -30, 32, -6, 12, -1, -41, 2, -4], 
'66':[-32, -11, -3, 37, -44, 1, -13, 34, -41, -41, -38, -28, 39, 6, 33, 0, 17, 34, 44], 
'58':[-10, 0, -4, -32, 0, 0, 44, -8, 35, 35, 31, 0, -31, 23, -36, 0, 33, -3, -2], 
'56':[-33, 0, -18, 40, -2, 0, 9, 32, 31, 31, 36, 0, -27, 3, -32, 0, 16, 10, 13], 
'54':[46, 0, 15, 13, 32, 0, -4, 7, 0, 0, -17, 0, -32, 39, -38, 0, 16, 9, 7], 
'46':[-33, 7, 1, 37, -33, -34, -33, -33, -32, -32, 37, -32, 37, 36, 35, 0, 34, 12, 8], 
'44':[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
'42':[33, 33, 0, 6, 33, 20, 7, 20, 4, 4, 0, 33, -35, 3, -35, -18, 10, -10, -9], 
}




for key in bpy.data.groups['Keys'].objects:  #reset keys to off mode
    key.location.z = 0


for driver in drivers:  #resets drivers their default positions
    driver[0].location.z = 0


for frame in range(frames): #for each frame
    for msg in port.iter_pending():  #Update beingPressed
        if hasattr(msg, 'velocity'):  #if it is a note (rather than modulation wheel for example)
            if msg.velocity == 0:  #if velocity is 0, that means the key has just stopped being pressed
                beingPressed.remove(msg.note) 
                key = bpy.data.objects[str(msg.note)]
                key.location.z = 0
            else:
                beingPressed.append(msg.note)
                key = bpy.data.objects[str(msg.note)]
                key.location.z = msg.velocity
                
            if msg.note == 84 and msg.velocity > 0: #84 is the key that switches the character
                faceDrivers[currentFace%len(faceDrivers)].location.z = 0 #reset previous face shape key driver
                currentFace += 1
                faceDrivers[currentFace%len(faceDrivers)].location.z = 1 #turn on new shape key driver
            
            
    
    
    for note in beingPressed: #stuff for keys that set the face to a default pose
        if str(note) in poseKeys.keys():
            for i, driver in enumerate(drivers, start=0):
                driver[0].location.z = poseKeys[str(note)][i]
                
    for driver in drivers:  #stuff for keys that affect muscle drivers
        if beingPressed.__contains__(driver[1]):
            if driver[0].location.z < 30:
                hitStrength = bpy.data.objects[str(driver[1])].location.z
                driver[0].location.z += 3 + controlSpeed*hitStrength
        if beingPressed.__contains__(driver[2]):
            if driver[0].location.z > -30:
                hitStrength = bpy.data.objects[str(driver[2])].location.z
                driver[0].location.z -= 3 + controlSpeed*hitStrength
    
    #next frame
    bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
    sleep(.025)
    print((frame+1)/frames)




#reset positions of keys and drivers
for driver in faceDrivers:
    driver.location.z = 0
faceDrivers[0].location.z = 1


for key in bpy.data.groups['Keys'].objects:
    key.location.z = 0


for driver in drivers:
    driver[0].location.z = 0



2 Likes

OH MY GOODNESS!! That is both utterly hilarious and very very cool!! Great work, can’t believe nobody’s commented on this yet!!

Brilliant - I know lots of people are looking for this, you may get very popular shortly…

Cheers, Clock.

PS. I too cannot believe there are not more comments here.