php code:

<?php

function themename_admin_enqueue_scripts() {
	wp_register_style( 'themename-admin-style', get_template_directory_uri() . '/css/themename-admin.css', array(), '1.0', 'all' );
	wp_enqueue_style( 'themename-admin-style' );

}
add_action( 'admin_enqueue_scripts', 'themename_admin_enqueue_scripts' );

// ===================== btn =====================

function themename_add_btn_button() {
	// Don't bother doing this stuff if the current user lacks permissions
	if ( ! current_user_can('edit_posts') && ! current_user_can('edit_pages') )
		return;

	if ( get_user_option( 'rich_editing' ) == 'true' ) { // add only in Rich Editor mode
		add_filter( 'mce_external_plugins', 'themename_add_btn_tinymce_plugin' );
		add_filter( 'mce_buttons', 'themename_register_btn_button' );
	}
}

function themename_register_btn_button( $buttons ) {
	array_push( $buttons, "|", "btn" );
	return $buttons;
}

function themename_add_btn_tinymce_plugin( $plugin_array ) { // load the TinyMCE plugin - plugin_embed.js
	$plugin_array&#91;'btn'&#93; = get_template_directory_uri() . '/js/shortcode_btn.js';
	return $plugin_array;
}

function themename_btn_refresh_mce( $ver ) {
	$ver += 3;
	return $ver;
}

// init process for button control
add_filter( 'tiny_mce_version', 'themename_btn_refresh_mce' );
add_action( 'init', 'themename_add_btn_button' );



function themename_btn_shortcode( $atts ) {
	$return = '';
	extract( shortcode_atts( array(
		'text' => '',
		'link' => ''
	), $atts ) );
	$return .= '<a class="btn btn-custom" href="'.$link.'">'.$text.'</a>';
	return $return;
}
add_shortcode( 'btn', 'themename_btn_shortcode' );
?>

javascript code:

// closure to avoid namespace collision
(function(){
	// creates the plugin
	tinymce.create('tinymce.plugins.btn', {
		init : function(ed, url) { // creates control instances based on the control's id
			ed.addButton('btn', {
			title : 'Button Shortcode', // title of the button
			image : url+'/../img/tinymce_btn.png', // path to the button's image
			onclick : function() { // triggers the thickbox
				tb_show( 'Tabs Shortcode', '#TB_inline?width=600&height=400&inlineId=btn-form' );
				jQuery('#TB_ajaxContent').css({'width': '640', 'height': (jQuery('#TB_window').height()-50)+'px'});
				jQuery(window).resize(function(){
					jQuery('#TB_ajaxContent').css({'width': '640', 'height': (jQuery('#TB_window').height()-50)+'px'});
				});
		}
	});
	}
	});
	
	tinymce.PluginManager.add('btn', tinymce.plugins.btn); // registers the plugin
	
	
	jQuery(function(){ // executes this when the DOM is ready
		// creates a form to be displayed everytime the button is clicked
		// you should achieve this using AJAX instead of direct html code like this
		var form = jQuery('<div id="btn-form">\
		<p class="popup_submit_wrap"><input type="button" id="btn-submit" class="button-primary" value="Insert Button" name="submit" /></p>\
		<div class="popup_content">\
            <p><strong><label for="btn-text">Button text</label></strong><br>\
            <input type="text" class="widefat" id="btn-text" name="text" value="" /></p>\
            <p><strong><label for="btn-link">Button link</label></strong><br>\
            <input type="text" class="widefat" id="btn-link" name="link" value="" /></p>\
		</div>\
		</div>');
		
		var popup_content = form.find('.popup_content');
		form.appendTo('body').hide();
		
		form.find('#btn-submit').click(function(){ // handles the click event of the submit button

			var shortcode = '[btn';

            var btn_text = popup_content.find('#btn-text').val();
            var btn_link = popup_content.find('#btn-link').val();

            shortcode += ' text="'+btn_text+'"';
            shortcode += ' link="'+btn_link+'"';

			shortcode += ']';
			
			tinyMCE.activeEditor.execCommand('mceInsertContent', 0, shortcode); // inserts the shortcode into the active editor
			
			tb_remove(); // closes Thickbox
		});
	});
})()

css code:

/* ============== shortcodes ============== */
.popup_submit_wrap {
	position: fixed;
	display: block;
	margin: -2px 0 0!important;
	width: 625px;
	height: 30px;
	padding: 20px 0!important;
	text-align: right;
	background-color: rgba(255,255,255,0.7);
	border-bottom: 1px solid #cfcfcf;
	z-index: 20;
}

.popup_content {
	position: relative;
	padding: 85px 0 20px;
}