Project | Synthesia + Piano Clone

Project | Synthesia + Piano Clone

For my project I worked on creating a clone for Synthesia.

Why?

I destroyed an electronic piano once by taking it apart and thinking I could fix it.

I couldn't. That moment has haunted my sensibilities for the longest time.

I could've picked an easier or more sensible project but sometimes me has to confront the ghosts of our past.

Challenges

  • Woeful Fabrication Skills: I've stabbed and burned myself multiple times in the making of this piano. I've even had to get a tetanus shot because out of frustration I ended up using subpar hacks including a rusty thumb pin.
  • Fabrication: The fabrication of the piano was a real limiting factor since it took me about 10-20 hours just making the piano keys and experimenting with the mechanisms
  • Multiplexing: Figure out how to properly multiplex the values. Currently ran into limitations with circuit design where multiple keys did not register. There's also a problem where the values weren't being read according to resister. I suspect that the resistors for each key may have been linked somehow in a series circuit instead of being parallel circuits and I'll have to study parallel circuit design properly to implement it, or atlernatively being able to
  • Properly Debouncing: The code for data read-ins and for triggering of the keys run multiple times. Ideally I'd want to modify the code to properly detect keyPressed and keyReleased to implement a smoouth sound function
  • Adding sound properly: I removed the sound because not being able to ease in and out of a note and coupled with the debouncing issues caused

Possible Improvements

  • Scope: Add up to 32 keys
  • Fabrication: Identify how to laser cut the key mechanisms properly. Improve the mechanism by
  • Circuit Design: Get a multiplexing module to support multiple keys
  • Code: Improve debouncing, write better drivers to detect key press and key release. Add sound.
image

image
image
image
image
image
image
image
image

Appendix

Arduino Sketch Code

#include <Encoder.h>

// Rotary Encoder Inputs
#define CLK A0
#define DT A1
#define SW A2
#define PUSHBTN 2

void setup() {
   Serial.begin(9600);
   pinMode(2, INPUT); 
   pinMode(3, INPUT); 
   pinMode(4, INPUT); 
   pinMode(5, INPUT); 
   pinMode(6, INPUT); 
//   while (Serial.available() <= 0) {
//     Serial.println("hello"); // send a starting message
//     delay(300);              // wait 1/3 second
//   }
}

int multiplexer1;
int multiplexer2;

void loop() {
   if (Serial.available()) {
      // read the incoming byte:
      int inByte = Serial.read();
      
      // read the sensor:
      multiplexer1 = analogRead(A0);
      multiplexer2 = analogRead(A1);
      // print the results:
      Serial.print(digitalRead(2)); Serial.print(",");
      Serial.print(digitalRead(3)); Serial.print(",");
      Serial.print(digitalRead(4)); Serial.print(",");
      Serial.print(digitalRead(5)); Serial.print(",");
      Serial.print(digitalRead(6));
      Serial.println();
   }
  delay(1);
}

P5 Sketch Code

let piano;
let debug;
let possibleNotes = [65, 83, 68, 70, 71];
let serialHelper;
let inData = "Awaiting Connection...."
let osc

function setup() {
    createCanvas(600, 600)
    piano = new PianoSynth();

    // Create a helper for managing p5 Serial Processing
    serialHelper = new SerialHelper({
        autoSelectPortName: true,
        onData: (data) => {
            if (data !== undefined && data !== "") {
                inData = data;
                // Handle key press
                fill('black');
                ellipse(50, 50, 50)
                if (data.indexOf("1") !== -1) {
                    inData = inData.split(",").map(i => parseInt(i)).map((v, i) => v === 1 ? possibleNotes[i] : v);
                    piano.keyPressed();
                }
            }
        },
        pollFn: (serial) => {
            serial.write(byte('\n'))
        },
        pollInterval: 1000 / 60,
    });

    // A triangle oscillator
    osc = new p5.TriOsc();
    // Start silent
    osc.start();
    osc.amp(0);
}

function draw() {
    background(240);

    frameRate(60);
    noStroke();

    text(`${keyCode}, ${keyIsPressed}`, 450, 65)
    piano.draw();

    if (keyIsPressed) {
        piano.keyPressed();
    }

}

class PianoSynth {
    gap = 60;
    width = 40;
    unit = 20;
    notes = [];

    constructor() {
        this.startingPoint = 0;
        this.globalTime = 0;

        this.notes = _.times(50).map(i => {
            let selectedNote = random(possibleNotes.filter((i,v) => v !== 3));
            let selectedNoteIndex = possibleNotes.indexOf(selectedNote);

            let l = random([1, 2, 3]);
            return {
                id: selectedNoteIndex,
                t: (i * 4) + (l),
                length: l,
                key: selectedNote
            }
        })
        fill('black');
    }

    draw() {
        text(`${keyCode}, ${keyIsPressed}`, 450, 50)
        text(`${inData}`, 450, 125)

        // Draw each note
        this.notes.forEach((note, i) => {
            let x = (note.id * this.gap);
            let y = this.startingPoint + this.globalTime - (note.t * this.unit);
            let l = this.unit * note.length;

            // Update the position
            note.x = x;
            note.y = y;

            // Draw the note
            fill('blue');
            rect(x, y, 30, l);

            // Add Text
            fill('white');
            text(char(note.key), x + 10, y + 20);

            fill('black');
        })

        if (this.notesPending().length === 0) {
            this.moveDown();
        }

        if (inData !== undefined && inData !== "" && inData instanceof Array) {
            inData.forEach((v, i) => {
                fill(color(0, 150));

                if (v !== 0) {
                    ellipse((i * 60) + 20, height, 25)
                }
                fill('black')
            })

            const notes = [60, 62, 64, 65, 67, 69, 71];
            inData.map((v, i) => v === 0 ? -1 : possibleNotes.indexOf(v)).filter((v) => v !== -1).forEach(key => {
                playNote(notes[key], 50);
            });
        }
    }

    moveDown(t = 1) {
        this.globalTime = this.globalTime + t;
    }

    notesPending() {
        let notesAtBottom = [];

        // Remove any note off screen;
        this.notes = this.notes.filter(note => note.y < height)

        this.notes.forEach((note, i) => {
            // Ignore if the note is off screen
            if (note.y > height) {
                return;
            }

            let d = dist(note.x, note.y, note.x, height)
            // Or the distance is less than the size
            if (d < this.unit * note.length) {
                notesAtBottom.push(note);
            }
        })

        notesAtBottom.forEach((note, i) => {
            text(`${note.y}, ${note.key}, ${height}`, 450, 90 + (i * 30))
        })

        return notesAtBottom;
    }

    correctNoteBeingPlayed() {
        // Each note that should be pressed is being pressed
        let notesPending = this.notesPending().map(i => i.key);
        let correctNotes = _.intersection(possibleNotes, notesPending);
        let incorrectNotes = _.difference(possibleNotes, notesPending);
        if (correctNotes.length === 0) {
            return false;
        }

        // Only correct notes are played
        let keyIsPlayed = key => this.keyIsDown(key);
        let correctNotesPlayed = _.every(correctNotes, keyIsPlayed);
        return correctNotesPlayed
    }

    keyIsDown(key) {
        let notesPlayed = inData.filter(v => v !== 0);
        if (notesPlayed.indexOf(key) !== -1) {
            return true;
        }
        return keyIsDown(key)
    }

    keyPressed() {
        if (this.correctNoteBeingPlayed()) {
            this.moveDown();
        }
    }
}


function playNote(note, duration) {
    osc.freq(midiToFreq(note));
    // Fade it in
    osc.fade(0.5, 0.2);

    // If we sest a duration, fade it out
    if (duration) {
        setTimeout(function () {
            osc.fade(0, 0.2);
        }, duration - 50);
    }

    setTimeout(function () {
        osc.fade(0, 0.5);
    }, duration);

}

// Fade it out when we release
function mouseReleased() {
    
}

P5 Helper Code

class SerialHelper {
    constructor(opts = {}) {
        this.opts = opts;
        this.pollInterval = opts.pollInterval || 300;
        // Requires portName
        // - Otherwise don't create the object
        if (!opts.portName) {
            console.log("INFO: No portname provided.")

            // List serial ports & return
            // - Autoselects if that configuration is opted for
            const serial = new p5.SerialPort();
            serial.on('list', this.onList.bind(this));
            serial.list();
            return
        }

        // Run as usual
        // ------------

        // Baud Rate
        const portName = opts.portName
        const baudRate = opts.baudRate || 9600;

        // Create the serial helper object
        this.serial = new p5.SerialPort();

        // Attach functions for
        this.serial.on('connected', this.onConnected.bind(this));
        this.serial.on('open', this.onOpen.bind(this));
        this.serial.on('data', _.throttle(this.onData.bind(this), 6, {
            leading: true,
            trailing: false
        })),
        this.serial.on('error', this.onError.bind(this));
        this.serial.on('close', this.onClose.bind(this));

        // Open the port
        this.serial.open(portName, {
            baudRate: baudRate
        });

        // Request data every 300 miliseconds
        setInterval(() => {
            this.serial.write("\n");
        }, this.pollInterval);
    }

    onList(portList) {
        // Display the list of ports
        portList.forEach((port, i) => {
            console.log(`${i} | ${port}`);
        })

        // Select default portname (using magic)
        const opts = this.opts;
        if (opts.autoSelectPortName) {
            this.handleAutomaticPortSelection(portList);
        } else {
            console.log("ERR: Please specify a portname from the list below.");
            return; // Break - Since we cannot determine portname
        }

        return portList;
    }

    handleAutomaticPortSelection(portList) {
        const opts = this.opts;
        const validPortNames = portList.filter(i => i.indexOf('tty.usbmodem') >= 0);
        console.log(`INFO: Selecting portname automatically from ${validPortNames}`)
        if (validPortNames.length > 0) {
            opts.portName = validPortNames[0];
            this.serial = new SerialHelper(opts);
            console.log(`INFO: Selected portname ${opts.portName}`)
        } else {
            console.log("ERR: Autoselect Failed. Please specify a portname from the list below.");
            return; // Break - Since we cannot determine portname
        }
    }

    onConnected() {
        if (isValidFunction(this.opts, 'onConnected')) {
            this.opts.onConnected();
            this.serial.clear();
        } else {
            console.log('INFO: onConnected not defined | Running default')
            console.log('INFO: The serial port opened.')
        }
    }

    onOpen() {
        if (isValidFunction(this.opts, 'onOpen')) {
            this.opts.onOpen();
            this.serial.clear();
        } else {
            console.log('INFO: onOpen not defined | Running default')
            console.log('INFO: Connected to server. Clearing serial buffer...');
            this.serial.clear();
        }
    }

    onData() {
        if (isValidFunction(this.opts, 'onData')) {
            let inData = this.serial.readLine();
            this.opts.onData(inData);
        } else {
            console.log('INFO: onData not defined | Running default')
        }
    }

    onError() {
        if (isValidFunction(this.opts, 'onError')) {2
            this.opts.onError();
        } else {
            console.log('INFO: onError not defined | Running default')
        }
    }

    onClose() {
        if (isValidFunction(this.opts, 'onClose')) {
            this.opts.onClose();
        } else {
            console.log('INFO: onClose not defined | Running default')
        }
    }
}

function isValidFunction(obj, fnName) {
    return obj[fnName] !== undefined && obj[fnName] instanceof Function
}